Index: branches/eam_branches/20090522/DataStore/Changes =================================================================== --- branches/eam_branches/20090522/DataStore/Changes (revision 24529) +++ branches/eam_branches/20090522/DataStore/Changes (revision 24557) @@ -7,4 +7,5 @@ - change ds*ls utilties to always return the HTTP status code - 300 as the exit status + - add --xattr flag to dsget 0.07 Mon May 12 12:04:24 HST 2008 Index: branches/eam_branches/20090522/DataStore/scripts/dsget =================================================================== --- branches/eam_branches/20090522/DataStore/scripts/dsget (revision 24529) +++ branches/eam_branches/20090522/DataStore/scripts/dsget (revision 24557) @@ -1,7 +1,5 @@ #!/usr/bin/env perl -# Copyright (C) 2006-2008 Joshua Hoblitt -# -# $Id: dsget,v 1.35 2008-09-24 21:58:03 jhoblitt Exp $ +# Copyright (C) 2006-2009 Joshua Hoblitt use strict; @@ -9,5 +7,5 @@ use vars qw( $VERSION ); -$VERSION = '0.02'; +$VERSION = '0.03'; use DataStore; @@ -28,5 +26,6 @@ $server, $compress, - $timeout + $timeout, + @xattrs, ); @@ -43,5 +42,6 @@ 'filename|f=s' => \$filename, 'compress|c' => \$compress, - 'timeout|t=s' => \$timeout, + 'timeout|t=s' => \$timeout, + 'xattr|x=s' => \@xattrs, ) or pod2usage( 2 ); @@ -159,4 +159,6 @@ require Nebulous::Client; import Nebulous::Client; + require Nebulous::Util; + import Nebulous::Util qw( :standard ); require File::Copy; import File::Copy; @@ -188,4 +190,9 @@ or die "Nebulous can't set xattr on key $filename"; + if (scalar @xattrs) { + write_xattrs($neb, $filename, @xattrs) + or die "Nebulous set xattrs on key $filename"; + } + open(my $src_fh, $tmpfilename) or die "can't open file $tmpfilename: $!"; @@ -230,5 +237,6 @@ dsget --uri --filename [--bytes ] [--md5 ] - [--compress] [--nebulous] [--volume ] [--copies ] + [--compress] [--nebulous] [--xattr user.:] + [--volume ] [--copies ] =head1 DESCRIPTION @@ -265,4 +273,11 @@ Optional. +=item * --xattr|-x + +Flag to set a Nebulous xattr on C<--filename>. Can be specified more than +once. + +Optional. + =item * --compress|-c @@ -306,5 +321,5 @@ =head1 COPYRIGHT -Copyright (C) 2006-2008 Joshua Hoblitt. All rights reserved. +Copyright (C) 2006-2009 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/eam_branches/20090522/DataStore/t/07_file.t =================================================================== --- branches/eam_branches/20090522/DataStore/t/07_file.t (revision 24529) +++ branches/eam_branches/20090522/DataStore/t/07_file.t (revision 24557) @@ -115,5 +115,5 @@ uri => "http://localhost:$port/somefile", fileid => '12buckelyourshoe', - bytes => 9, + bytes => 10, md5sum => 'a0a6e1a375117c58d77221f10c5ce12e', type => 'chip', @@ -129,5 +129,5 @@ uri => "http://localhost:$port/somefile", fileid => '12buckelyourshoe', - bytes => 9, + bytes => 10, md5sum => 'a0a6e1a375117c58d77221f10c5ce12e', type => 'chip', Index: branches/eam_branches/20090522/DataStoreServer/scripts/dsprodindex =================================================================== --- branches/eam_branches/20090522/DataStoreServer/scripts/dsprodindex (revision 24529) +++ branches/eam_branches/20090522/DataStoreServer/scripts/dsprodindex (revision 24557) @@ -80,5 +80,5 @@ } -$stmt = $dbh->prepare("SELECT * FROM dsFileset WHERE prod_id = $prod_id $after_clause"); +$stmt = $dbh->prepare("SELECT * FROM dsFileset WHERE prod_id = $prod_id $after_clause ORDER BY fileset_id"); $stmt->execute(); Index: branches/eam_branches/20090522/Nebulous-Server/Build.PL =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/Build.PL (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/Build.PL (revision 24557) @@ -10,4 +10,5 @@ requires => { 'Apache2::Const' => 0, + 'Cache::Memcached' => 0, 'Class::Accessor::Fast' => 0, 'Config::YAML' => '1.42', Index: branches/eam_branches/20090522/Nebulous-Server/Changes =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/Changes (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/Changes (revision 24557) @@ -2,4 +2,8 @@ 0.17 + - add basic valid key caching via memcached + - add params checking to Nebulous::Key::parse_neb_key() + - restrict creation/modification/remove of xattrs to the user.* namespace + - add chmod_object() method - retry database transactions when a deadlock is detected - add log4perl logging to nebdiskd @@ -16,4 +20,13 @@ - change 'log_level' param to 'trace' - refactor ->find_objects() functionality + - rename Nebulous::Key->soft_volume Nebulous::Key->hard_volume and invert + the semantics + - add nebdiskd mountpoint test retyring + - rework delete_instance() to avoid requiring an index on instance.uri + - nebdiskd bug fixes: fix debug mode outpoint going to mail, remove db + passwd requirement (not all dbs require a password param) + - completely rework how mountedvol is populated, drop mount table + - infinitely try to get a db handle if the connection fails + - add --mountpoint param to neb-voladd 0.16 Index: branches/eam_branches/20090522/Nebulous-Server/MANIFEST =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/MANIFEST (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/MANIFEST (revision 24557) @@ -33,18 +33,19 @@ t/01_load.t t/02_config.t -t/02_server_setup.t -t/03_server_create_object.t -t/04_server_replicate_object.t -t/05_server_lock_object.t -t/06_server_unlock_object.t -t/07_server_find_instances.t -t/08_server_delete_instance.t -t/09_server_stat_object.t -t/10_server_is_valid_volume_name.t -t/11_server_is_valid_object_key.t -t/12_server_find_objects.t -t/13_server_rename_object.t -t/14_server_xattr.t -t/15_mounts.t -t/16_server_swap_objects.t +t/03_server_setup.t +t/04_server_create_object.t +t/05_server_replicate_object.t +t/06_server_lock_object.t +t/07_server_unlock_object.t +t/08_server_find_instances.t +t/09_server_delete_instance.t +t/10_server_stat_object.t +t/11_server_is_valid_volume_name.t +t/12_server_is_valid_object_key.t +t/13_server_find_objects.t +t/14_server_rename_object.t +t/15_server_xattr.t +t/16_mounts.t +t/17_server_swap_objects.t +t/18_server_chmod_object.t t/75_parse_neb_key.t Index: branches/eam_branches/20090522/Nebulous-Server/bin/neb-voladd =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/bin/neb-voladd (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/bin/neb-voladd (revision 24557) @@ -19,5 +19,5 @@ use Pod::Usage qw( pod2usage ); -my ($db, $dbhost, $dbuser, $dbpass, $vname, $vhost, $uri); +my ($db, $dbhost, $dbuser, $dbpass, $mountpoint, $vname, $vhost, $uri); $db = $ENV{'NEB_DB'} unless $db; @@ -29,14 +29,15 @@ 'db|d=s' => \$db, 'host=s' => \$dbhost, + 'mountpoint|m=s' => \$mountpoint, + 'pass|p=s' => \$dbpass, + 'uri|u=s' => \$uri, 'user|u=s' => \$dbuser, - 'pass|p=s' => \$dbpass, + 'vhost=s' => \$vhost, 'vname|n=s' => \$vname, - 'vhost=s' => \$vhost, - 'uri|u=s' => \$uri, ) || pod2usage( 2 ); pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass --vname --vhost --uri", -exitval => 2 ) - unless $db && $dbuser && $dbpass && $vname && $vhost && $uri; +pod2usage( -msg => "Required options: --db --user --pass --mountpoint --vname --vhost --uri", -exitval => 2 ) + unless $db && $dbuser && $dbpass && $mountpoint && $vname && $vhost && $uri; my $dbh = DBI->connect( @@ -59,4 +60,7 @@ die "path: $path dirived from URI: $uri does not exist"; } +unless (-d $mountpoint) { + die "path: $mountpoint does not exist"; +} print " OK\n"; @@ -65,5 +69,5 @@ my $query = $dbh->prepare( $sql->new_volume ); -$query->execute( $vname, $vhost, $path ); +$query->execute( $vname, $vhost, $path, $mountpoint); print " OK\n"; @@ -128,4 +132,7 @@ Optional. Defaults to C. +=item * --mountpoint + +Path to the mountpoint of the filesystem on which C<--uri> resides. =back Index: branches/eam_branches/20090522/Nebulous-Server/bin/nebdiskd =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/bin/nebdiskd (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/bin/nebdiskd (revision 24557) @@ -1,7 +1,5 @@ #!/usr/bin/env perl -# Copyright (C) 2007 Joshua Hoblitt -# -# $Id: nebdiskd,v 1.14 2008-10-16 22:51:34 jhoblitt Exp $ +# Copyright (C) 2007-2009 Joshua Hoblitt use strict; @@ -35,8 +33,9 @@ $user, $group, + $retry, ); GetOptions( - 'dbhost|h=s' => \$dbhost, + 'dbhost|H=s' => \$dbhost, 'dbpass|p=s' => \$dbpass, 'dbuser|u=s' => \$dbuser, @@ -46,6 +45,7 @@ 'pidfile=s' => \$pidfile, 'restart|r' => \$restart, + 'retry=i' => \$retry, 'stop|s' => \$stop, - 'user=s' => \$user, + 'user|U=s' => \$user, 'verbose|v' => \$verbose, ) || pod2usage( 2 ); @@ -61,10 +61,11 @@ $dbhost ||= $c->get_dbhost || $ENV{'NEB_DBHOST'} || 'localhost'; $dbuser ||= $c->get_dbuser || $ENV{'NEB_USER'}; -$dbpass ||= $c->get_dbpass || $ENV{'NEB_PASS'}; +$dbpass ||= $c->get_dbpass || $ENV{'NEB_PASS'} || undef; $pidfile ||= $c->get_pidfile || "/var/tmp/nebdiskd"; $user ||= $c->get_user || $<; # user $group ||= $c->get_group || $); # group - -my $mounts = $c->get_mounts; +$retry ||= $c->get_retry || 1; + +#my $mounts = $c->get_mounts; my $poll_interval = $c->get_poll_interval || 5; @@ -80,9 +81,10 @@ $c->set_pidfile($pidfile); $c->set_poll_interval($poll_interval); +$c->set_retry($retry); $c->write; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --db --user --pass", -exitval => 2 ) - unless $db && $dbuser && $dbpass; +pod2usage( -msg => "Required options: --db --dbuser", -exitval => 2 ) + unless $db && $dbuser; # start up logging @@ -108,7 +110,12 @@ log4perl.appender.Mailer.to = ps-ipp-ops@ifa.hawaii.edu log4perl.appender.Mailer.subject = nebdiskd alert - log4perl.appender.AppError.Filter= MatchWarn + log4perl.appender.Mailer.buffered = 0 + log4perl.appender.Mailer.Filter= MatchWarn log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n + + log4perl.appender.Limiter = Log::Log4perl::Appender::Limit + log4perl.appender.Limiter.appender = Mailer + log4perl.appender.Limiter.block_period = 300 '; Log::Log4perl::init(\$conf); @@ -136,4 +143,5 @@ poll_interval => $poll_interval, debug => $debug, + retry => $retry, ); }; @@ -155,7 +163,8 @@ my $dbhost = $p{dbhost} or return; my $dbuser = $p{dbuser} or return; - my $dbpass = $p{dbpass} or return; +# my $dbpass = $p{dbpass} or return; my $poll_interval = $p{poll_interval} || 60; my $debug = $p{debug} || 0; + my $retry = $p{retry} || 1; # setup the db on every pass incase the database died on us @@ -168,20 +177,41 @@ eval { - my $r_query = $dbh->prepare_cached("REPLACE INTO mount VALUES(?, ?, ?)"); - my $d_query = $dbh->prepare_cached("DELETE FROM mount WHERE mountpoint = ?"); - + my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT *, ?, ? FROM volume WHERE mountpoint = ?"); + my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?"); + + # get list of mount points + my $mounts = []; + { + # there may be multiple vol_ids per mountpoint but we only need to + # check each mointpont once + my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume"); + $query->execute; + while (my $row = $query->fetchrow_hashref) { + push @$mounts, $row; + } + $query->finish; + } + # determine valid mountpoints foreach my $mnt (@$mounts) { - $log->debug("checking $mnt"); + my $mountpoint = $mnt->{'mountpoint'}; + $log->debug("checking $mountpoint"); # this /SHOULD/ fail if the mount point is handled by the # automounter and it fails to mount - eval { - unless (is_mountpoint($mnt)) { + my $tries = 0; + TEST: eval { + $tries++; + unless (is_mountpoint($mountpoint)) { $log->warn("$mnt is not a valid mountpoint"); } }; if ($@) { + # try is_mountpoint() again if $retry > 1 + if ($tries < $retry) { + $log->warn("retrying test of $mountpoint"); + goto TEST; + } $log->warn($@); - $d_query->execute($mnt); + $d_query->execute($mountpoint); next; } @@ -190,23 +220,24 @@ # we determine if it's a valid mountpoint incase # is_mountpoint() invokes the automounter - my $dev_info = df($mnt, 1024); + my $dev_info = df($mountpoint, 1024); unless (defined $dev_info) { - $log->error("can't find device info for $mnt"); + $log->error("can't find device info for $mountpoint"); next; } - $r_query->execute($mnt, @$dev_info{qw( blocks used )}); - $log->debug("adding $mnt to db"); + # find vol_id(s) for mountpoint + $r_query->execute(@$dev_info{qw( blocks used )}, $mountpoint); + $log->debug("adding $mountpoint to db"); } - $dbh->do("call getmountedvol()"); - - $dbh->commit; - $log->debug("commited to database"); +# $dbh->do("call getmountedvol()"); + +# $dbh->commit; +# $log->debug("commited to database"); }; if ($@) { - $dbh->rollback; - $log->debug("rolledback transaction"); +# $dbh->rollback; +# $log->debug("rolledback transaction"); $log->logdie($@); } @@ -235,8 +266,9 @@ my %p = @_; + # $p{dbpass} may be undef; + return unless defined $p{db} and defined $p{dbhost} - and defined $p{dbuser} - and defined $p{dbpass}; + and defined $p{dbuser}; my $sql = Nebulous::Server::SQL->new; @@ -249,5 +281,5 @@ RaiseError => 1, PrintError => 1, - AutoCommit => 0, + AutoCommit => 1, # don't want this to be trasnactional }, ); @@ -255,8 +287,8 @@ eval { $dbh->do( $sql->set_transaction_model ); - $dbh->commit; +# $dbh->commit; }; if ($@) { - $dbh->rollback; +# $dbh->rollback; $log->logdie($@); } @@ -344,5 +376,8 @@ =head1 SYNOPSIS - nebdiskd [--db ] [--user ] [--pass ] [--debug] [--pidfile ] [--stop] [--restart] [--verbose] + nebdiskd [--db|-D ] [--dbhost|-H + [--dbuser|-u ] [--dbpass|-p ] [--debug|-d] + [--user|-U ] [--group|-g ] [--pidfile ] + [--retry ] [--stop|-s] [--restart|-r] [--verbose|-v] =head1 DESCRIPTION @@ -355,5 +390,5 @@ =over 4 -=item * --db|-d +=item * --db|-D Name of database (C) to write too. @@ -362,5 +397,5 @@ variable is set. -=item * --user|-u +=item * --dbuser|-u Username to authenticate with. @@ -369,12 +404,19 @@ variable is set. -=item * --pass|-p +=item * --dbpass|-p Password to authenticate with. + +Optional if defined in the F<.nebdiskdrc> file or the appropriate environment +variable is set or if the database does not require a password. + +=item * --dbhost|-H + +Database host to connect to. Optional if defined in the F<.nebdiskdrc> file or the appropriate environment variable is set. -=item * --debug +=item * --debug|-d This flag prevents the program from "daemonizing" so output can be sent to @@ -397,5 +439,10 @@ instance. -=item * --verbose|-r +=item * --retry + +The number of times to test a mountpoint for "mountedness" before giving up on +it. + +=item * --verbose|-v Turns on informational/debugging messages to be sent to the @@ -403,4 +450,12 @@ with C<--debug>. +=item * --user|-U + +user account to run daemon as. + +=item * --group|-g + +group to run daemon as. + =back @@ -415,13 +470,13 @@ =item * C -Equivalent to --db|-d +Equivalent to --db|-D =item * C -Equivalent to --user|-u +Equivalent to --dbuser|-u =item * C -Equivalent to --pass|-p +Equivalent to --dbpass|-p =back @@ -439,10 +494,7 @@ dbpass: '@neb@' dbuser: nebulous - mounts: - - /mnt - - /tmp - - /usr pidfile: /var/tmp/nebdiskd poll_interval: 5 + retry: 3 The values C, C, C, and C have the same semantics @@ -451,10 +503,4 @@ =over 4 -=item * C - -A list of "paths" to C before mounted volumes are polled. The propose -of this option is to attempt to keep "automounted" volumes mounted while this -program is running. - This value may be omitted or left blank. @@ -465,4 +511,8 @@ This value may be omitted or left blank. The default value is C<60>s. +=item * C + +Same as C<--retry>. + =back @@ -481,5 +531,5 @@ =head1 COPYRIGHT -Copyright (C) 2007 Joshua Hoblitt. All rights reserved. +Copyright (C) 2007-2009 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Key.pm =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Key.pm (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Key.pm (revision 24557) @@ -8,5 +8,5 @@ use warnings FATAL => qw( all ); -our $VERSION = '0.02'; +our $VERSION = '0.03'; use base qw( Exporter Class::Accessor::Fast ); @@ -14,4 +14,5 @@ use File::Spec; use URI::file; +use Carp qw( croak ); use URI; use overload '""' => \&_stringify_key; @@ -22,14 +23,16 @@ ); -__PACKAGE__->mk_ro_accessors(qw( path volume soft_volume )); +__PACKAGE__->mk_ro_accessors(qw( path volume hard_volume )); sub parse_neb_key { my ($key, $volume) = @_; - return unless defined $key; + + croak "key param is not optional" unless defined $key; + croak "too many params" if scalar @_ > 2; # white space is not allowed if ($key =~ qr/\s+/) { - die "keys and URIs may not contain whitespace"; + croak "keys and URIs may not contain whitespace"; } @@ -39,9 +42,9 @@ my $volume_name; - my $soft_volume; + my $hard_volume; # if this is a valid uri if (defined $scheme) { # if so, does it use the neb scheme? - die "URI does not use the 'neb' scheme" + croak "URI does not use the 'neb' scheme" unless $scheme eq 'neb'; @@ -58,5 +61,5 @@ # path component to it unless (length $path) { - die "neb URI scheme requires a path component"; + croak "neb URI scheme requires a path component"; } @@ -68,10 +71,10 @@ if (defined $volume_info->{volume} and $volume_info->{volume} ne 'any') { $volume_name = $volume_info->{volume}; - $soft_volume = $volume_info->{soft_volume}; + $hard_volume = $volume_info->{hard_volume}; } # require a leading slash if there is no volume name if ((not defined $volume_name) and (not $path =~ m|^/|)) { - die "neb URI scheme requires a leading slash, eg. neb:/"; + croak "neb URI scheme requires a leading slash, eg. neb:/"; } } else { @@ -81,5 +84,5 @@ if (defined $volume_info->{volume} and $volume_info->{volume} ne 'any') { $volume_name = $volume_info->{volume}; - $soft_volume = $volume_info->{soft_volume}; + $hard_volume = $volume_info->{hard_volume}; } } @@ -96,5 +99,5 @@ return __PACKAGE__->new({ volume => $volume_name, - soft_volume => $soft_volume, + hard_volume => $hard_volume, path => $path, }); @@ -107,11 +110,11 @@ return unless defined $volume; - my $soft_volume; + my $hard_volume; # check to see if there is a tilde and remove it if found - unless (defined $volume and $volume =~ s/^~//) { - $soft_volume = 1; + if (defined $volume and $volume =~ s/^~//) { + $hard_volume = 1; } - return({ volume => $volume, soft_volume => $soft_volume }); + return({ volume => $volume, hard_volume => $hard_volume }); } @@ -122,8 +125,8 @@ my $path = $self->path; - my $volume = $self->volume || ""; - my $soft_volume = $self->soft_volume ? '~' : ""; - - return "neb://${soft_volume}${volume}/$path"; + my $volume = $self->volume || ''; + my $hard_volume = $self->hard_volume ? '~' : ''; + + return "neb://${hard_volume}${volume}/$path"; } Index: branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server.pm =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server.pm (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server.pm (revision 24557) @@ -13,6 +13,8 @@ use base qw( Class::Accessor::Fast ); +use Cache::Memcached; use DBI; use Digest::SHA1 qw( sha1_hex ); +use Fcntl ':mode'; use File::Basename qw( basename dirname fileparse ); use File::ExtAttr qw( setfattr ); @@ -27,8 +29,13 @@ use URI::file; -__PACKAGE__->mk_accessors(qw( log sql config )); - -use constant SUBPATH_DEPTH => 2; -use constant NFS_RETRIES => 100; +__PACKAGE__->mk_accessors(qw( log sql config cache )); + +use constant SUBPATH_DEPTH => 2; +use constant NFS_RETRIES => 100; +use constant NFS_RETRY_WAIT => 1; +use constant TRANS_RETRY_WAIT => 1; + +# transaction restart/retry regex +my $trans_regex = qr/Deadlock Found|Lock wait timeout exceeded|try restarting transaction|Can't connect to MySQL server/i; sub new @@ -45,5 +52,7 @@ sub new_from_config { - my ($class, $config) = @_; + my $class = shift; + + my ($config) = @_; # log4perl is not avaliable until we call init() @@ -51,8 +60,7 @@ my $log = Log::Log4perl::get_logger( "Nebulous::Server" ); $log->level($config->trace); + $log->debug( "entered - @_" ); my $sql = Nebulous::Server::SQL->new; - - $log->debug( "entered - @_" ); my $self = bless {}, ref $class || $class; @@ -60,4 +68,15 @@ $self->sql($sql); $self->config($config); + $self->cache( + Cache::Memcached->new({ + servers => $config->memcached_servers, + }) + ); +# $self->cache->set("foo", "bar") or die "set failed"; +# $self->cache->get("foo") or die "get failed"; + $log->logdie("at least one database must be defined") unless $config->n_db; + + # cause a db session to be started + $self->_db_for_index(0); $log->debug( "leaving" ); @@ -70,10 +89,15 @@ sub _db_index_for_key { - my ($self, $key) = @_; + my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); + + my ($key) = @_; my $config = $self->config; my $db_index = 0; - die "key not defined" unless defined $key; + $log->logdie("key not defined") unless defined $key; # hash the key to select the correct database instance @@ -85,4 +109,7 @@ $db_index = unpack("h8", sha1_hex($path)) % $config->n_db; + $log->debug("index is $db_index"); + $log->debug("leaving"); + return $db_index; } @@ -90,7 +117,11 @@ sub _db_for_index { - my ($self, $db_index) = @_; + my $self = shift; my $log = $self->log; + $log->debug( "entered - @_" ); + + my ($db_index) = @_; + my $sql = $self->sql; my $config = $self->config; @@ -108,5 +139,5 @@ # lookup database info my $db_config = $config->db($db_index); - die "can't find database configuration info for database # $db_index" + $log->logdie("can't find database configuration info for db # $db_index") unless $db_config; @@ -116,29 +147,39 @@ # processes and the database might have gone away on us. Apache::DBI will # take care of getting a valid dbh back. - eval { - $dbh = DBI->connect_cached( - $db_config->dsn, - $db_config->dbuser, - $db_config->dbpasswd, - { - RaiseError => 1, - PrintError => 0, - AutoCommit => 0, - }, - ); - - $dbh->do( $sql->set_transaction_model ); - $log->debug( "connected to database: ", sub { $dbh->data_sources; } ); - $dbh->commit; - $log->debug("commit"); - }; - if ( $@ ) { - $dbh->rollback if $dbh; - $log->debug("rollback"); - $log->logdie( "database error: $@" ); + TRANS: while (1) { + eval { + $dbh = DBI->connect_cached( + $db_config->dsn, + $db_config->dbuser, + $db_config->dbpasswd, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 0, + }, + ); + + $dbh->do( $sql->set_transaction_model ); + $log->debug( "connected to database: ", sub { $dbh->data_sources; } ); + $dbh->commit; + $log->debug("commit"); + }; + if ($@) { + $dbh->rollback if $dbh; + $log->debug("rollback") if $dbh; + if ($@ =~ qr/Can't connect to MySQL server/) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; + redo TRANS; + } + $log->logdie( "database error: $@" ); + } + last; } $self->{dbs}[$db_index] = $dbh; + $log->debug("leaving"); + return $dbh; } @@ -148,4 +189,7 @@ my $self = shift; + my $log = $self->log; + $log->debug( "entered - @_" ); + my ($key) = validate_pos(@_, { @@ -154,12 +198,12 @@ ); - my $log = $self->log; my $sql = $self->sql; my $config = $self->config; - die "key not defined" unless defined $key; + $log->logdie("key not defined") unless defined $key; my $db_index = $self->_db_index_for_key($key); my $dbh = $self->_db_for_index($db_index); + return $dbh; } @@ -168,4 +212,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); my ($key, $vol_name) = validate_pos(@_, @@ -186,13 +233,14 @@ ); + my $sql = $self->sql; + # vol_name overrides the key implied volume - $key = parse_neb_key($key, $vol_name); + eval { + $key = parse_neb_key($key, $vol_name); + }; + $log->logdie("$@") if $@; $vol_name = $key->volume; - my $log = $self->log; - my $sql = $self->sql; my $db = $self->db($key); - - $log->debug( "entered - @_" ); # the key's volume can't be validiated on input for this method so we have @@ -200,14 +248,14 @@ if (defined $vol_name and not $self->_is_valid_volume_name($key, $key->volume)) { - if ($key->soft_volume) { + unless ($key->hard_volume) { $log->warn( "$vol_name is not a known volume name" ); $vol_name = undef; } else { - die "$vol_name is not a valid volume name" + $log->logdie("$vol_name is not a valid volume name"); } } my ($vol_id, $vol_host, $vol_path, $vol_xattr) - = $self->_get_storage_volume($key, $vol_name, $key->soft_volume); + = $self->_get_storage_volume($key, $vol_name, $key->hard_volume); my $parent_id = $self->_resolve_dir_parent_id(key => $key, create => 1); @@ -282,6 +330,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -291,4 +340,8 @@ } + # add new key to the cache + $self->cache->set($key->path, 1) if defined $self->cache; + $log->debug( "key added to cache" ); + $log->debug("leaving"); @@ -300,4 +353,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); my %p = validate(@_, @@ -322,9 +378,6 @@ my $key = $p{key}; - my $log = $self->log; my $sql = $self->sql; my $db = $self->db($key); - - $log->debug( "entered - @_" ); # no path means '/', which has a dir_id & parent_id of 1 @@ -394,8 +447,8 @@ $query->finish; } - $log->logdie("failed to get LAST_INSERT_ID()") + die("failed to get LAST_INSERT_ID()") unless $parent_id; - $db->commit; +# $db->commit; } }; @@ -403,8 +456,16 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + $parent_id = 1; + sleep TRANS_RETRY_WAIT; redo TRANS; } + if ($@ =~ qr/Duplicate entry/) { + $log->warn("Duplicate database entry, retrying transaction: $@"); + $parent_id = 1; + sleep TRANS_RETRY_WAIT; + redo TRANS; + } $log->logdie("error: $@"); } @@ -421,4 +482,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); my ($key, $newkey) = validate_pos(@_, @@ -438,13 +502,17 @@ ); + my $sql = $self->sql; + # ignore volumes - $key = parse_neb_key($key); - $newkey = parse_neb_key($newkey); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + eval { + $newkey = parse_neb_key($newkey); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug("entered - @_"); # XXX this may require database migration in the future @@ -464,6 +532,9 @@ unless ($rows == 1) { $query->finish; - $log->logdie("affected row count is $rows instead of 1"); - } + die("affected row count is $rows instead of 1"); + } + + $self->cache->delete($key->path) if defined $self->cache; + $self->cache->set($newkey->path, 1) if defined $self->cache; $db->commit; @@ -473,6 +544,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -491,4 +563,7 @@ my $self = shift; + my $log = $self->log; + $log->debug("entered - @_"); + my ($key1, $key2) = validate_pos(@_, { @@ -506,20 +581,25 @@ ); + my $sql = $self->sql; + # ignore volumes - $key1 = parse_neb_key($key1); - $key2 = parse_neb_key($key2); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key1 = parse_neb_key($key1); + }; + $log->logdie("$@") if $@; + eval { + $key2 = parse_neb_key($key2); + }; + $log->logdie("$@") if $@; my $dbidx1 = $self->_db_index_for_key($key1); my $dbidx2 = $self->_db_index_for_key($key2); - die "cannot swap keys not stored on the same database" unless ($dbidx1 == $dbidx2); + $log->logdie("cannot swap keys not stored on the same database") + unless ($dbidx1 == $dbidx2); my $dbh1 = $self->_db_for_index($dbidx1); my $dbh2 = $self->_db_for_index($dbidx2); - die "different db handles for the same db?" unless ($dbh1 == $dbh2); - - $log->debug("entered - @_"); + $log->logdie("different db handles for the same db?") + unless ($dbh1 == $dbh2); # order of operations for the swap with a single db is: @@ -543,5 +623,5 @@ unless ($rows == 1) { $query->finish; - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } } @@ -556,5 +636,5 @@ unless ($rows == 1) { $query->finish; - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } } @@ -569,5 +649,5 @@ unless ($rows == 1) { $query->finish; - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } } @@ -579,6 +659,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -612,4 +693,7 @@ my $self = shift; + my $log = $self->log; + $log->debug("entered - @_"); + my ($key, $vol_name) = validate_pos(@_, { @@ -633,4 +717,6 @@ ); + my $sql = $self->sql; + # if a volume name is explicity specified then we should make the # replication onto that volume (even if there is alread an instance on that @@ -642,20 +728,19 @@ # vol_name overrides the key implied volume - $key = parse_neb_key($key, $vol_name); + eval { + $key = parse_neb_key($key, $vol_name); + }; + $log->logdie("$@") if $@; $vol_name = $key->volume; - my $log = $self->log; - my $sql = $self->sql; my $db = $self->db($key); - - $log->debug("entered - @_"); if (defined $vol_name and not $self->_is_valid_volume_name($key, $key->volume)) { - if ($key->soft_volume) { + unless ($key->hard_volume) { $log->warn( "$vol_name is not a known volume name" ); $vol_name = undef; } else { - die "$vol_name is not a valid volume name" + $log->logdie("$vol_name is not a valid volume name"); } } @@ -681,5 +766,5 @@ unless ( $rows > 0 ) { $query->finish; - $log->logdie( "storage object does not exist" ); + die( "storage object does not exist" ); } @@ -725,11 +810,12 @@ $db->rollback; # handle soft volumes - if (defined $vol_name and defined $key->soft_volume) { + if (defined $vol_name and not defined $key->hard_volume) { $log->debug("retrying with 'any' volume"); return $self->replicate_object($key->path, 'any'); } $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -739,4 +825,17 @@ } + # check to see if the user.mode xattr exists + eval { + my $mode = $self->getxattr_object("$key", 'user.mode'); + if (defined $mode) { + $self->chmod_object("$key", $mode); + } + }; + if ($@) { + unless ($@ =~ qr/user.mode does not exist/) { + $log->logdie("error: $@"); + } + } + $log->debug("leaving"); @@ -748,4 +847,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); my ( $key, $type ) = validate_pos( @_, @@ -764,12 +866,13 @@ ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug( "entered - @_" ); my $so_id; @@ -785,5 +888,5 @@ unless ( $rows == 1 ) { $query->finish; - $log->logdie( "storage object does not exist" ); + die( "storage object does not exist" ); } @@ -800,9 +903,9 @@ # can't set a write lock if there are read locks if ($write_lock) { - $log->logdie("can not write lock twice -- retry"); + die("can not write lock twice -- retry"); } if ($read_lock > 0) { - $log->logdie("can not write lock after read lock -- retry"); + die("can not write lock after read lock -- retry"); } @@ -813,5 +916,5 @@ # if we affected more then one row something very bad has happened. unless ($rows == 1) { - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } @@ -820,5 +923,5 @@ # can't set a read lock if there's a write lock if ($write_lock) { - $log->logdie("can not read lock after write lock -- retry"); + die("can not read lock after write lock -- retry"); } @@ -829,5 +932,5 @@ # if we affected more then one row something very bad has happened. unless ($rows == 1) { - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } } @@ -840,6 +943,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -858,4 +962,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); my ( $key, $type ) = validate_pos( @_, @@ -874,12 +981,13 @@ ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug( "entered - @_" ); my $so_id; @@ -895,5 +1003,5 @@ unless ($rows == 1) { $query->finish; - $log->logdie("storage object does not exist"); + die("storage object does not exist"); } @@ -909,9 +1017,9 @@ # can't remove a write lock if it doesn't exist if ($read_lock) { - $log->logdie("can not have a write lock under a read lock"); + die("can not have a write lock under a read lock"); } unless ($write_lock) { - $log->logdie("can not remove non-existant write lock"); + die("can not remove non-existant write lock"); } @@ -922,5 +1030,5 @@ # if we affected more then one row something very bad has happened. unless ($rows == 1) { - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } } @@ -929,9 +1037,9 @@ # can't remove a read lock if there aren't any if ($write_lock) { - $log->logdie("can not have a read lock under a write lock"); + die("can not have a read lock under a write lock"); } if ($read_lock == 0) { - $log->logdie("can not remove non-existant read lock"); + die("can not remove non-existant read lock"); } @@ -942,5 +1050,5 @@ # if we affected more then one row something very bad has happened. unless ($rows == 1) { - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } @@ -953,6 +1061,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -971,4 +1080,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); my ($key, $name, $value, $flags) = validate_pos(@_, @@ -981,4 +1093,8 @@ { type => SCALAR, + callbacks => { + 'xattr is in user. namespace' + => sub { ($_[0]) =~ qr/^user\./ }, + }, }, { @@ -993,12 +1109,13 @@ ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug("entered - @_"); TRANS: while (1) { @@ -1020,5 +1137,5 @@ if ($flags eq 'create') { unless ($rows == 1) { - $log->logdie( "affected row count is $rows instead of 1" ); + die( "affected row count is $rows instead of 1" ); } } else { @@ -1026,5 +1143,5 @@ # the case of a replace and 1 if the xattr didn't already exist. unless ($rows == 1 or $rows == 2) { - $log->logdie( "affected row count is $rows instead of 2" ); + die( "affected row count is $rows instead of 2" ); } } @@ -1036,6 +1153,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -1054,4 +1172,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); my ($key, $name) = validate_pos(@_, @@ -1064,15 +1185,20 @@ { type => SCALAR, + callbacks => { + 'xattr is in user. namespace' + => sub { ($_[0]) =~ qr/^user\./ }, + }, }, ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug("entered - @_"); my $value; @@ -1085,10 +1211,10 @@ if ($rows == 0) { $query->finish; - $log->logdie( "xattr $key:$name does not exist" ); + die( "xattr $key:$name does not exist" ); } # if we go more then one row bad something very bad has happened. unless ($rows == 1) { $query->finish; - $log->logdie( "affected row count is $rows instead of 1" ); + die( "affected row count is $rows instead of 1" ); } @@ -1099,5 +1225,11 @@ $value = $row->{ 'value' }; }; - $log->logdie("database error: $@") if $@; + if ($@) { + if ($@ =~ /user\..*? does not exist/) { + # do not log xattr does not exist messages + die $@; + } + $log->logdie("database error: $@") if $@; + } $log->debug("leaving"); @@ -1110,4 +1242,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); my ($key) = validate_pos(@_, @@ -1120,12 +1255,13 @@ ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug("entered - @_"); my @xattrs; @@ -1151,4 +1287,7 @@ my $self = shift; + my $log = $self->log; + $log->debug("entered - @_"); + my ($key, $name) = validate_pos(@_, { @@ -1160,15 +1299,20 @@ { type => SCALAR, + callbacks => { + 'xattr is in user. namespace' + => sub { ($_[0]) =~ qr/^user\./ }, + }, }, ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug("entered - @_"); TRANS: while (1) { @@ -1179,7 +1323,12 @@ $query->finish; + # no rows affected means the xattr did not exist + if ($rows == 0) { + die( "xattr $key:$name does not exist" ); + } + # if we affected more then one row something very bad has happened. - unless ($rows == 1) { - $log->logdie( "affected row count is $rows instead of 1" ); + if ($rows > 1) { + die( "affected row count is $rows instead of 1" ); } @@ -1190,6 +1339,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -1209,4 +1359,7 @@ my $self = shift; + my $log = $self->log; + $log->debug( "entered - @_" ); + my ($pattern) = validate_pos( @_, { @@ -1216,9 +1369,9 @@ ); - $pattern = parse_neb_key($pattern); - - my $log = $self->log; - - $log->debug( "entered - @_" ); + + eval { + $pattern = parse_neb_key($pattern) if defined $pattern; + }; + $log->logdie("$@") if $@; unless (defined $pattern) { @@ -1235,4 +1388,11 @@ $log->logdie("no keys found") unless ( scalar @keys ); + if (defined $self->cache) { + foreach my $path (@keys) { + $self->cache->set($path, 1); + $log->debug("key added to cache as: $path"); + } + } + $log->debug( "leaving" ); @@ -1245,12 +1405,14 @@ my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); + my $index = shift; my $key = shift; - my $log = $self->log; my $sql = $self->sql; my $db = $self->_db_for_index($index); - $log->debug( "entered - @_" ); # first check to see if the key is an exact match @@ -1310,4 +1472,7 @@ my $self = shift; + my $log = $self->log; + $log->debug("entered - @_"); + my ($key, $vol_name) = validate_pos(@_, { @@ -1330,13 +1495,19 @@ ); + my $sql = $self->sql; + +# unless ($key) { +# $log->warn("key was undefined after validate_pos(), trying again..."); +# return $self->find_instances(@_); +# } + # vol_name overrides the key implied volume - $key = parse_neb_key($key, $vol_name); + eval { + $key = parse_neb_key($key, $vol_name); + }; + $log->logdie("$@") if $@; $vol_name = $key->volume; - my $log = $self->log; - my $sql = $self->sql; my $db = $self->db($key); - - $log->debug("entered - @_"); # the key's volume can't be validiated on input for this method so we have @@ -1344,9 +1515,9 @@ if (defined $vol_name and not $self->_is_valid_volume_name($key, $key->volume)) { - if ($key->soft_volume) { + if ($key->hard_volume) { + $log->logdie("$vol_name is not a valid volume name"); + } else { $log->warn( "$vol_name is not a known volume name" ); $vol_name = undef; - } else { - die "$vol_name is not a valid volume name" } } @@ -1361,5 +1532,5 @@ unless ($rows > 0) { $query->finish; - $log->logdie("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name"); + die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name"); } } else { @@ -1369,5 +1540,5 @@ unless ($rows > 0) { $query->finish; - $log->logdie("no instances available for key: $key"); + die("no instances available for key: $key"); } } @@ -1381,5 +1552,5 @@ $db->rollback; # handle soft volumes - if (defined $vol_name and defined $key->soft_volume) { + if (defined $vol_name and not defined $key->hard_volume) { $log->debug("retrying with 'any' volume"); return $self->find_instances($key->path, 'any'); @@ -1403,4 +1574,7 @@ my $self = shift; + my $log = $self->log; + $log->debug( "entered - @_" ); + my ($key, $uri) = validate_pos( @_, { @@ -1415,37 +1589,44 @@ ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug( "entered - @_" ); TRANS: while (1) { eval { + my $instances; my $so_id; - my $instances; - # get so_id + my $ins_id; + # find so_id for key and get count of instances { - my $query = $db->prepare_cached( $sql->get_object_from_uri ); - my $rows = $query->execute( $uri ); - + my $query = $db->prepare_cached( $sql->get_instance_count_by_ext_id ); + my $rows = $query->execute($key->path); unless ( $rows > 0 ) { $query->finish; - $log->logdie( "no instance is associated with uri" ); - } - - $so_id = $query->fetchrow_hashref->{ 'so_id' }; + die( "$key has no associated instances - this should not happen" ); + } + + my $record = $query->fetchrow_hashref; + $so_id = $record->{ 'so_id' }; + $instances = $record->{ 'count(ins_id)' }; $query->finish; - - } - + } + + # find ins_id for uri { - my $query = $db->prepare_cached( $sql->get_instance_count ); - $query->execute( $so_id ); - - $instances = $query->fetchrow_hashref->{ 'count(ins_id)' }; + my $query = $db->prepare_cached( $sql->get_instance_by_uri ); + my $rows = $query->execute($so_id, $uri); + unless ( $rows > 0) { + $query->finish; + die( "no instance is associated with uri" ); + } + + $ins_id = $query->fetchrow_hashref->{ 'ins_id' }; $query->finish; } @@ -1453,12 +1634,12 @@ # remove instance { - my $query = $db->prepare_cached( $sql->delete_instance ); - my $rows = $query->execute( $uri ); + my $query = $db->prepare_cached( $sql->delete_instance_by_ins_id ); + my $rows = $query->execute( $ins_id ); $query->finish; - # if we affected something other then two rows something very bad - # has happened + # if we affected something other then one row something very + # bad has happened unless ( $rows == 1 ) { - $log->logdie( "affected row count is $rows instead of 1" ); + die( "affected row count is $rows instead of 1" ); } } @@ -1467,4 +1648,7 @@ # remove it too if ( $instances == 1 ) { + # remove key from cache + $self->cache->delete($key->path) if defined $self->cache; + # we just removed the last instance my $query = $db->prepare_cached( $sql->delete_object ); @@ -1474,5 +1658,5 @@ # TODO: this will have to be changed in order to support hardlinks unless ( $rows == 1 ) { - $log->logdie( "affected row count is $rows instead of 2" ); + die( "affected row count is $rows instead of 2" ); } } @@ -1484,6 +1668,7 @@ $db->rollback; $log->debug("rollback"); - if ($@ =~ /Deadlock found/) { - $log->warn("database deadlock retrying transaction: $@"); + if ($@ =~ $trans_regex) { + $log->warn("database error, retrying transaction: $@"); + sleep TRANS_RETRY_WAIT; redo TRANS; } @@ -1502,4 +1687,7 @@ { my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); my ( $key ) = validate_pos( @_, @@ -1512,12 +1700,13 @@ ); + my $sql = $self->sql; + # ignore volume - $key = parse_neb_key($key); - - my $log = $self->log; - my $sql = $self->sql; + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + my $db = $self->db($key); - - $log->debug("entered - @_"); my $stat; @@ -1527,5 +1716,5 @@ unless ($rows == 1) { - $log->logdie("no storage object found"); + die("no storage object found"); } @@ -1548,12 +1737,12 @@ my $self = shift; + my $log = $self->log; + $log->debug("entered - @_"); + validate_pos(@_); - my $log = $self->log; my $sql = $self->sql; my $db = $self->_db_for_index(0); # XXX fix as above - $log->debug("entered - @_"); - my $stats; my $query; @@ -1576,4 +1765,81 @@ } +sub chmod_object +{ + my $self = shift; + + my $log = $self->log; + $log->debug("entered - @_"); + + my ($key, $mode) = validate_pos( @_, + { + type => SCALAR, + callbacks => { + 'is valid object key' => sub { $self->_is_valid_object_key($_[0]) }, + }, + }, + { + type => SCALAR, + regex => qr/\d{3,4}/, + callbacks => { + 'is allowable mode' => sub { + $_[0] == (S_IRUSR | S_IRGRP) + }, + }, + }, + ); + + my $sql = $self->sql; + + # ignore volume + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + + my $db = $self->db($key); + + # find all instances of this object + my $locations; + eval { + $locations = $self->find_instances("$key"); + }; + $log->logdie("error: $@") if $@; + + # update each instances + foreach my $inst (@$locations) { + my $path = URI->new($inst)->path; + + $self->_retry(sub { chmod($mode, $path) }) + or $log->logdie("can not chmod() $path: $!"); + + # XXX I'm assuming that it's OK to fsync() a filehandle that's only + # open for reading? Opening as w/rw here can fail if the chmod removes + # write permissions. + my $fh; + $self->_retry(sub { open($fh, '<', $path) }) + or $log->logdie("can not open() $path: $!"); + + # fsync(3c) + $self->_retry(sub { $fh->sync() }) + or $log->logdie("can not sync() $path: $!"); + + $self->_retry(sub { close($fh) }) + or $log->logdie("can not close() $path: $!"); + } + + # stick an xattr on this object with the mode + # XXX this would probably be better as a field in the storage_object_attr + # table but since we're not planning to use this for very many objects (as + # a %) it may not be worth adding the extra field at this time. + eval { + $self->setxattr_object("$key", 'user.mode', $mode, 'replace'); + }; + $log->logdie("error: $@") if $@; + + $log->debug("leaving"); + + return $mode; +} sub _get_storage_volume @@ -1581,13 +1847,13 @@ my $self = shift; - my ($key, $name, $soft_volume) = @_; - - my $log = $self->log; - my $sql = $self->sql; - my $db = $self->db($key); - + my $log = $self->log; no warnings qw( uninitialized ); $log->debug( "entered - @_" ); use warnings; + + my ($key, $name, $hard_volume) = @_; + + my $sql = $self->sql; + my $db = $self->db($key); my ($vol_id, $vol_host, $vol_path, $xattr); @@ -1605,14 +1871,14 @@ # if a volume name was specified, and is soft, and we failed to # find it, fall back to any volume - if ($soft_volume) { + unless ($hard_volume) { ($vol_id, $vol_host, $vol_path, $xattr) = $self->_get_storage_volume($key); return; # this just returns out of the eval not from the subroutine } - $log->logdie("storage volume: $name is not available"); + die("storage volume: $name is not available"); } # when matching by name we shouldn't ever match more than once if ($rows > 1) { $query->finish; - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } } else { @@ -1623,5 +1889,5 @@ unless ($rows > 0) { $query->finish; - $log->logdie("no storage volume is available"); + die("no storage volume is available for key: $key volume: $name hard_volume: $hard_volume"); } } @@ -1631,5 +1897,14 @@ $query->finish; }; - $log->logdie("database error: $@") if $@; + if ($@) { + if ($@ =~ qr/no storage volume is available/) { + # this should not happen unless all volumes are full + $log->error($@); + $log->debug("retrying..."); + return $self->_get_storage_volume(@_); + } + # else + $log->logdie("database error: $@"); + } $log->logdie("failed to find a suitable volume" ) @@ -1646,13 +1921,13 @@ my $self = shift; - my $key = shift; - - my $log = $self->log; - my $sql = $self->sql; - my $db = $self->db($key); - + my $log = $self->log; no warnings qw( uninitialized ); $log->debug( "entered - @_" ); use warnings; + + my $key = shift; + + my $sql = $self->sql; + my $db = $self->db($key); my ($vol_id, $vol_host, $vol_path, $xattr); @@ -1665,10 +1940,10 @@ unless ($rows > 0) { $query->finish; - $log->logdie("can't find a suitable storage volume to replicate $key to"); + die("can't find a suitable storage volume to replicate $key to"); } # when matching by name we shouldn't ever match more than once if ($rows > 1) { $query->finish; - $log->logdie("affected row count is $rows instead of 1"); + die("affected row count is $rows instead of 1"); } @@ -1690,10 +1965,28 @@ sub _is_valid_object_key { - my ($self, $key) = @_; - - $key = parse_neb_key($key); - - my $log = $self->log; + my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); + + my ($key) = @_; + my $sql = $self->sql; + + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; + + # check cache first + my $cached = $self->cache->get($key->path) if defined $self->cache; + if (defined $cached) { + $log->debug( "key $key found in cache as ", $key->path ); + $log->debug( "leaving" ); + return 1; + } else { + $log->debug( "key $key not found in cache" ); + } + my $db = $self->db($key); @@ -1712,7 +2005,15 @@ if (defined $ext_id) { + $log->debug( "key found in db" ); + # add key to cache + $self->cache->set($key->path, 1) if defined $self->cache; + $log->debug( "key added to cache as ", $key->path ); + $log->debug( "leaving" ); return 1; } + $log->debug( "key not found in db" ); + $log->debug( "leaving" ); + return; } @@ -1721,11 +2022,21 @@ sub _is_valid_volume_name { - my ($self, $key, $vol_name) = @_; - - $key = parse_neb_key($key); + my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); + + my ($key, $vol_name) = @_; + + my $sql = $self->sql; + + # the volume name implied by the key is ignored. $key is only needed to + # select a database connection + eval { + $key = parse_neb_key($key); + }; + $log->logdie("$@") if $@; my $volume_info = parse_neb_volume($vol_name); - my $log = $self->log; - my $sql = $self->sql; my $db = $self->db($key); @@ -1734,4 +2045,6 @@ # handle "any" volume if ($vol_name eq 'any') { + $log->debug( "found volume name $vol_name" ); + $log->debug( "leaving" ); return 1; } @@ -1748,7 +2061,12 @@ if (defined $vol_id and defined $vol_path) { + $log->debug( "found volume name $vol_name" ); + $log->debug( "leaving" ); return 1; } + $log->debug( "volume name $vol_name not found" ); + $log->debug( "leaving" ); + return; } @@ -1759,7 +2077,9 @@ my $self = shift; + my $log = $self->log; + $log->debug( "entered - @_" ); + my ($key, $so_id, $ins_id, $vol_path, $xattr) = @_; - my $log = $self->log; my $sql = $self->sql; my $db = $self->db($key); @@ -1770,13 +2090,15 @@ my $storage_filename = $self->_generate_storage_filename($key->path, $ins_id); unless (-d $storage_path) { - _retry(sub { mkpath(@_) }, $storage_path, 0, 0775) - or die "can't create storage path: $storage_path"; + $self->_retry(sub { mkpath([$storage_path], 0, 0775) }) + or die("can't create storage path: $storage_path"); } # check to make sure at least the parent directory has the proper # permissions - my $mode = [_retry(sub { stat($storage_path) } )]->[2] & 07777; + my $mode = [$self->_retry(sub { stat($storage_path) } )]->[2] & 07777; unless ($mode == 0775) { - $log->error("$storage_path has the wrong permissions of: %04x", $mode); - _retry(sub { chmod(0775, $storage_path) }) or die "can't chmod $storage_path: $!"; + # XXX: this problem is so common that it's flooding the logs + $log->debug("$storage_path has the wrong permissions of: 0", sprintf("%o", $mode)); + $self->_retry(sub { chmod(0775, $storage_path) }) + or die("can not chmod() $storage_path: $!"); } @@ -1787,4 +2109,8 @@ }; if ($@) { + if (defined $uri and -e $uri->file) { + unlink($uri->file) + or $log->error("failed to unlink() $uri: $!"); + } $log->logdie($@); } @@ -1792,5 +2118,5 @@ if ($xattr) { my $path = $uri->file; - die "can not set xattr on $path: $!" + $log->logdie("can not set xattr on $path: $!") unless (setfattr($path, 'user.nebulous_key', $key->path)); } @@ -1804,25 +2130,27 @@ my $self = shift; + my $log = $self->log; + $log->debug( "entered - @_" ); + my ($path) = @_; # perl's open() can't do an O_CREAT | O_EXCL - die "file $path already exists" if (-e $path); + # XXX is it possible to tell if this system call failed? + -e $path + and $log->logdie("file $path already exists"); my $fh; - die "can not open $path: $!" - unless (_retry(sub { open($fh, '>', $path) })); + $self->_retry(sub { open($fh, '>', $path) }) + or $log->logdie("can not open() $path: $!"); # chmod before fsync() to make sure the changed perms hit the disk too - die "can not chmod $path: $!" - unless (_retry(sub { chmod(0664, $path) })); - - die "can not flush $path: $!" - unless ($fh->flush); - - die "can not sync $path: $!" - unless ($fh->sync); - - die "can not close $path: $!" - unless (_retry(sub { close($fh) })); + $self->_retry(sub { chmod(0664, $path) }) + or $log->logdie("can not chmod() $path: $!"); + + $self->_retry(sub { $fh->sync() }) + or $log->logdie("can not sync() $path: $!"); + + $self->_retry(sub { close($fh) }) + or $log->logdie("can not close() $path: $!"); return $path; @@ -1861,4 +2189,9 @@ sub _retry { + my $self = shift; + + my $log = $self->log; + $log->debug( "entered - @_" ); + my $func = shift; @@ -1869,6 +2202,6 @@ }; if ($@) { - die $@; - sleep 1; + $log->logdie($@); + sleep NFS_RETRY_WAIT; next; } @@ -1879,5 +2212,5 @@ # if the loop ended and $@ is set, rethrow the error if ($@) { - die $@; + $log->logdie($@); } Index: branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server.pod =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server.pod (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server.pod (revision 24557) @@ -29,4 +29,5 @@ Nebulous::Server->stat_object( $key ); Nebulous::Server->mounts(); + Nebulous::Server->chmod_object( $key, $mode) ; =head1 DESCRIPTION @@ -203,4 +204,11 @@ ] +=item * chmod_object( $key, $mode ); + +Accepts 2 parameters, both mandatory. C<$key> is the nebulous key to operate +on and C<$mode> are the POSIX like file permissions to set on all instances of +C<$key>. C<$mode> must be specified in oct. Returns C<$mode> or sucess or an +exception on failure. + =back @@ -223,5 +231,5 @@ =head1 COPYRIGHT -Copyright (C) 2004-2008 Joshua Hoblitt. All rights reserved. +Copyright (C) 2004-2009 Joshua Hoblitt. All rights reserved. This program is free software; you can redistribute it and/or modify it under Index: branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server/Config.pm =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server/Config.pm (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server/Config.pm (revision 24557) @@ -13,5 +13,5 @@ use Log::Log4perl qw( :levels ); -use Params::Validate qw( validate validate_pos SCALAR ); +use Params::Validate qw( validate validate_pos SCALAR ARRAYREF ); our %LEVELS = ( @@ -46,7 +46,8 @@ }, }, - dsn => { type => SCALAR, optional => 1 }, - dbuser => { type => SCALAR, optional => 1 }, - dbpasswd => { type => SCALAR, optional => 1 }, + dsn => { type => SCALAR, optional => 1 }, + dbuser => { type => SCALAR, optional => 1 }, + dbpasswd => { type => SCALAR, optional => 1 }, + memcached_servers => { type => ARRAYREF, optional => 1 }, }; @@ -61,5 +62,8 @@ # normalize log levels to lower-case - my $self = { trace => $LEVELS{lc($p{trace})} }; + my $self = { + trace => $LEVELS{lc($p{trace})}, + memcached_servers => $p{memcached_servers}, + }; bless $self, $class || ref $class; Index: branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server/SQL.pm =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/lib/Nebulous/Server/SQL.pm (revision 24557) @@ -175,7 +175,7 @@ WHERE ext_id = ? }, - delete_instance => qq{ + delete_instance_by_ins_id => qq{ DELETE FROM instance - WHERE uri = ? + WHERE ins_id = ? }, get_object_from_uri => qq{ @@ -184,8 +184,24 @@ WHERE uri = ? }, + get_instance_by_uri => qq{ + SELECT ins_id + FROM instance + WHERE + so_id = ? + AND uri = ? + }, get_instance_count => qq{ SELECT count(ins_id) FROM instance WHERE so_id = ? + }, + get_instance_count_by_ext_id => qq{ + SELECT count(ins_id), so_id + FROM instance + JOIN storage_object + USING(so_id) + WHERE + ext_id = ? + GROUP BY so_id }, get_object_instances => qq{ @@ -233,41 +249,24 @@ get_replication_volume_for_ext_id => qq{ SELECT - vol_id, + m.vol_id, host, path, xattr, total - used as free - FROM ( --- This query works but is slow... --- SELECT --- mountedvol.* --- FROM mountedvol --- WHERE --- vol_id NOT IN( --- SELECT vol_id --- FROM storage_object --- JOIN instance --- USING(so_id) --- WHERE ext_id = -- --- ) - SELECT - m.* - FROM mountedvol AS m - LEFT JOIN instance AS i - ON m.vol_id = i.vol_id - AND i.so_id = ( - SELECT so_id - FROM storage_object - WHERE ext_id = ? - ) - WHERE - i.vol_id IS NULL - ) as Foo + FROM mountedvol AS m + LEFT JOIN instance AS i + ON m.vol_id = i.vol_id + AND i.so_id = ( + SELECT so_id + FROM storage_object + WHERE ext_id = ? + ) WHERE - used / total < ? + i.vol_id IS NULL + AND used / total < ? AND available = ? AND allocate = ? - ORDER BY RAND() - LIMIT 1 + ORDER BY RAND() + LIMIT 1 }, get_storage_volume => qq{ @@ -287,6 +286,6 @@ }, new_volume => qq{ - INSERT INTO volume (name, host, path, allocate, available, xattr) - VALUES (?, ?, ?, TRUE, TRUE, FALSE) + INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint) + VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?) }, get_volume_by_name => qq{ @@ -397,5 +396,5 @@ }, get_mounted_volumes => qq{ - SELECT * FROM mountedvol ORDER BY host, name + SELECT mountpoint, total, used, vol_id, name, host, path, allocate, available, xattr FROM mountedvol ORDER BY host, name }, ); @@ -424,7 +423,6 @@ DROP TABLE IF EXISTS lock_record; DROP TABLE IF EXISTS volume; -DROP TABLE IF EXISTS mount; +DROP TABLE IF EXISTS mountedvol; DROP TABLE IF EXISTS log; -DROP TABLE IF EXISTS mountedvol; DROP TABLE IF EXISTS directory; DROP PROCEDURE IF EXISTS getmountedvol; @@ -455,5 +453,7 @@ FOREIGN KEY(parent_id) REFERENCES directory(dir_id), PRIMARY KEY(dir_id), - KEY(parent_id) + KEY(parent_id), + KEY(dirname), + UNIQUE(dirname, parent_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -466,5 +466,5 @@ CREATE TABLE storage_object ( so_id BIGINT NOT NULL AUTO_INCREMENT, - ext_id VARCHAR(255) NOT NULL UNIQUE, + ext_id VARCHAR(255) NOT NULL, ext_id_basename VARCHAR(255) NOT NULL, dir_id BIGINT NOT NULL, @@ -472,4 +472,5 @@ type enum('REG_FILE'), PRIMARY KEY(so_id), + UNIQUE KEY(ext_id), KEY(dir_id), KEY(type) @@ -514,5 +515,5 @@ CREATE TABLE volume ( vol_id INT NOT NULL AUTO_INCREMENT, - name VARCHAR(255) UNIQUE NOT NULL, + name VARCHAR(255) NOT NULL, host VARCHAR(255) NOT NULL, path VARCHAR(255) NOT NULL, @@ -520,9 +521,38 @@ available BOOLEAN DEFAULT FALSE, xattr BOOLEAN DEFAULT FALSE, + mountpoint VARCHAR(255) NOT NULL, PRIMARY KEY(vol_id), + UNIQUE KEY(name), + UNiQUE KEY(path), KEY(host(16)), - KEY(path(255)), KEY(allocate), - KEY(available) + KEY(available), + KEY(mountpoint(255)) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +### + +CREATE TABLE mountedvol( + vol_id INT NOT NULL, + FOREIGN KEY(vol_id) REFERENCES volume(vol_id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + host VARCHAR(255) NOT NULL, + path VARCHAR(255) NOT NULL, + FOREIGN KEY(path) REFERENCES volume(path) ON DELETE CASCADE, + allocate BOOLEAN DEFAULT FALSE, + available BOOLEAN DEFAULT FALSE, + xattr BOOLEAN DEFAULT FALSE, + mountpoint VARCHAR(255) NOT NULL, + FOREIGN KEY(mountpoint) REFERENCES volume(mountpoint) ON DELETE CASCADE, + total BIGINT NOT NULL, + used BIGINT NOT NULL, + PRIMARY KEY(vol_id), + KEY(name), + KEY(host), + KEY(path), + KEY(allocate), + KEY(available), + KEY(xattr), + KEY(mountpoint(255)) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -546,13 +576,4 @@ ### -CREATE TABLE mount ( - mountpoint VARCHAR(255) NOT NULL, - total BIGINT NOT NULL, - used BIGINT NOT NULL, - PRIMARY KEY(mountpoint) -) ENGINE=innodb DEFAULT CHARSET=latin1; - -### - CREATE TABLE log ( timestamp TIMESTAMP, @@ -562,30 +583,4 @@ message VARCHAR(2048) NOT NULL, PRIMARY KEY(timestamp) -) ENGINE=innodb DEFAULT CHARSET=latin1; - -### - -CREATE TABLE mountedvol( - mountpoint VARCHAR(255) NOT NULL, - FOREIGN KEY(mountpoint) REFERENCES mount(mountpoint) ON DELETE CASCADE, - total BIGINT NOT NULL, - used BIGINT NOT NULL, - vol_id INT NOT NULL, - FOREIGN KEY(vol_id) REFERENCES volume(vol_id) ON DELETE CASCADE, - name VARCHAR(255) NOT NULL, - host VARCHAR(255) NOT NULL, - path VARCHAR(255) NOT NULL, - FOREIGN KEY(path) REFERENCES volume(path) ON DELETE CASCADE, - allocate BOOLEAN DEFAULT FALSE, - available BOOLEAN DEFAULT FALSE, - xattr BOOLEAN DEFAULT FALSE, - PRIMARY KEY(mountpoint), - KEY(vol_id), - KEY(name), - KEY(host), - KEY(path), - KEY(allocate), - KEY(available), - KEY(xattr) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -608,7 +603,7 @@ -- store the okey checking state - SELECT @@FOREIGN_KEY_CHECKS INTO key_checks; +-- SELECT @@FOREIGN_KEY_CHECKS INTO key_checks; -- disable foregin check checks to prevent deadlocks on the mountedvol table - SET FOREIGN_KEY_CHECKS=0; +-- SET FOREIGN_KEY_CHECKS=0; -- make sure the temp table does not already exist... this can happy if the @@ -629,10 +624,8 @@ OPEN cur1; - DELETE FROM mountedvol; - myloop: LOOP FETCH cur1 INTO vol_idvar, namevar, hostvar, pathvar, allocatevar, availablevar, xattrvar; IF `done` THEN LEAVE myloop; END IF; - INSERT INTO mountedvol + REPLACE INTO mountedvol SELECT mountpoint, total, used, vol_idvar, namevar, hostvar, pathvar, allocatevar, availablevar, xattrvar FROM @@ -650,9 +643,9 @@ -- restore the original key checking state - SET @@FOREIGN_KEY_CHECKS = key_checks; +-- SET @@FOREIGN_KEY_CHECKS = key_checks; DROP TABLE IF EXISTS myvolume; - SET FOREIGN_KEY_CHECKS=1; +-- SET FOREIGN_KEY_CHECKS=1; COMMIT; Index: branches/eam_branches/20090522/Nebulous-Server/lib/Test/Nebulous.pm =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/lib/Test/Nebulous.pm (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/lib/Test/Nebulous.pm (revision 24557) @@ -11,4 +11,5 @@ use base qw( Exporter ); +use Cache::Memcached; use DBI; use File::Path qw( mkpath rmtree ); @@ -16,9 +17,10 @@ use Nebulous::Server::SQL; -our @EXPORT = qw( $NEB_DB $NEB_USER $NEB_PASS ); +our @EXPORT = qw( $NEB_DB $NEB_USER $NEB_PASS $NEB_MEMCACHED_SERVERS); our $NEB_DB = $ENV{'NEB_DB'} || "DBI:mysql:database=test:host=localhost"; our $NEB_USER = $ENV{'NEB_USER'} || "test"; our $NEB_PASS = $ENV{'NEB_PASS'} || ''; +our $NEB_MEMCACHED_SERVERS = ['127.0.0.1:11211']; my $dbh = DBI->connect( $NEB_DB, $NEB_USER, $NEB_PASS ); @@ -53,34 +55,55 @@ # node01/node02: allocate = TRUE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (1, 'node01', 'node01', ?, TRUE, TRUE) }, undef, $dir1); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir1); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (1, 'node01', 'node01', ?, '/', TRUE, TRUE) }, undef, $dir1); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 1); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir1); - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (2, 'node02', 'node02', ?, TRUE, TRUE) }, undef, $dir2); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir2); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (2, 'node02', 'node02', ?, '/', TRUE, TRUE) }, undef, $dir2); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 2); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir2); # node03: allocate = TRUE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (3, 'node03', 'node03', ?, TRUE, TRUE) }, undef, $dir3); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir3); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (3, 'node03', 'node03', ?, '/', TRUE, TRUE) }, undef, $dir3); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e8 FROM volume WHERE vol_id = ? }, undef, 3); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e8) }, undef, $dir3); # node04: allocate = FALSE, available = FALSE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (4, 'node04', 'node04', ?, FALSE, FALSE) }, undef, $dir4); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir4); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (4, 'node04', 'node04', ?, '/', FALSE, FALSE) }, undef, $dir4); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 4); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir4); # node05: allocate = FALSE, available = TRUE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (5, 'node05', 'node05', ?, FALSE, TRUE) }, undef, $dir5); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir5); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (5, 'node05', 'node05', ?, '/', FALSE, TRUE) }, undef, $dir5); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 5); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir5); # node06: allocate = TRUE, available = FALSE - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (6, 'node06', 'node06', ?, TRUE, FALSE) }, undef, $dir6); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir6); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (6, 'node06', 'node06', ?, '/', TRUE, FALSE) }, undef, $dir6); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e7 FROM volume WHERE vol_id = ? }, undef, 6); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e7) }, undef, $dir6); # node07: full - $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, allocate, available) VALUES (7, 'node07', 'node07', ?, TRUE, TRUE) }, undef, $dir7); - $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e10) }, undef, $dir7); + $dbh->do(qq{ INSERT INTO volume (vol_id, name, host, path, mountpoint, allocate, available) VALUES (7, 'node07', 'node07', ?, '/', TRUE, TRUE) }, undef, $dir7); + $dbh->do(qq{ INSERT INTO mountedvol SELECT *, 10e10, 10e10 FROM volume WHERE vol_id = ? }, undef, 7); +# $dbh->do(qq{ INSERT INTO mount VALUES (?, 10e10, 10e10) }, undef, $dir7); - $dbh->do(qq{ call getmountedvol() }); +# $dbh->do(qq{ call getmountedvol() }); } sub cleanup { + # memcached needs to be emptied between test runs + Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS)->flush_all; + # if flush_all() turns out to be useless as feared +# my $memd = Cache::Memcached->new(servers => $NEB_MEMCACHED_SERVERS); +# my $query = $dbh->prepare("SELECT ext_id FROM storage_object"); +# if ($query->execute) { +# while (my $row = $query->fetchrow_hashref) { +# my $ext_id = $row->{ext_id}; +# warn "found key $ext_id\n" if $memd->get($ext_id); +# warn "deleted $ext_id\n" if $memd->delete($ext_id); +# } +# } +# $query->finish; + foreach my $statement (@{ $sql->get_db_clear }) { $dbh->do( $statement ); Index: branches/eam_branches/20090522/Nebulous-Server/scripts/dirize.pl =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/scripts/dirize.pl (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/scripts/dirize.pl (revision 24557) @@ -1,3 +1,6 @@ #!/usr/bin/env perl + +# query to check for duplicate directory entries +# select count(*) from (select so.so_id, so.ext_id, so.dir_id from storage_object as so join (select dir_id, dirname, parent_id, count(*) from directory group by dirname, parent_id having count(*) > 1) as foo on so.dir_id = foo.dir_id) as foo; use strict; @@ -23,17 +26,35 @@ my $db = $neb->_db_for_index(0); -my $n; -{ - my $query = $db->prepare("SELECT COUNT(*) as n FROM storage_object"); - $query->execute; - $n = $query->fetchrow_hashref->{'n'}; -} +my $n = 0; +#{ +# my $query = $db->prepare("SELECT COUNT(*) as n FROM storage_object"); +# $query->execute; +# $n = $query->fetchrow_hashref->{'n'}; +#} -my $query = $db->prepare_cached("SELECT so_id, ext_id, dir_id FROM storage_object AS so WHERE so.dir_id = 0 LIMIT 1000"); +# repair directory duplication +#my $query = $db->prepare_cached("select so.so_id, so.ext_id, so.dir_id from storage_object as so join (select dir_id, dirname, parent_id, count(*) from directory group by dirname, parent_id having count(*) > 1) as foo on so.dir_id = foo.dir_id limit 1000"); + +# initial directory fill in +my $work_query = $db->prepare_cached("SELECT so_id, ext_id, dir_id FROM storage_object AS so WHERE so.dir_id = 0 LIMIT 1000"); + +my $update_query = $db->prepare_cached("UPDATE storage_object SET ext_id_basename = ?, dir_id = ? WHERE so_id = ?"); + +# turn of fkeys, otherwise we can change storage_object.dir_id to a +# non-existant value +$db->do("SET FOREIGN_KEY_CHECKS=0"); +#$db->do("UPDATE storage_object SET dir_id = 0"); + +# completely reset the directory table +#$db->do("DELETE FROM directory"); +#$db->do("ALTER TABLE directory AUTO_INCREMENT = 1"); +# make sure these duplicates can't happen again +#$db->do("alter table directory add unique key(dirname,parent_id)"); +# seed the root ('/') directory +#$db->do("INSERT INTO directory VALUES(1, '/', 1)"); my $i = 0; -while ($query->execute and $query->rows) { - print "foo\n"; - while (my $row = $query->fetchrow_hashref) { +while ($work_query->execute and $work_query->rows) { + while (my $row = $work_query->fetchrow_hashref) { $i++; my $key = parse_neb_key($row->{'ext_id'}); @@ -41,10 +62,14 @@ #printf("dirizing %20s basename: %20s parent_id %10d\n", $key, basename($row->{'ext_id'}), $parent_id); - printf("$i/$n dirizing %s\n", $key->path); +# printf("$i dirizing %s\n", $key->path); - my $q = $db->prepare_cached("UPDATE storage_object SET ext_id_basename = ?, dir_id = ? WHERE so_id = ?"); - $q->execute(basename($row->{'ext_id'}), $parent_id, $row->{'so_id'}); - $db->commit; + $update_query->execute(basename($row->{'ext_id'}), $parent_id, $row->{'so_id'}); } - $query->finish; + $db->commit; + printf("### COMMIT ###\n"); + $work_query->finish; + printf("dirized $i\n"); } + +$db->do("SET FOREIGN_KEY_CHECKS=1"); +$db->commit; Index: branches/eam_branches/20090522/Nebulous-Server/t/02_config.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/02_config.t (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/t/02_config.t (revision 24557) @@ -8,5 +8,5 @@ use warnings; -use Test::More tests => 21; +use Test::More tests => 22; use lib qw( ./t ./lib ); @@ -109,2 +109,11 @@ is($config_db1->dbpasswd, "boo2", "dbpasswd"); } + +# memcached_servers +{ + my $config = Nebulous::Server::Config->new( + memcached_servers => ['127.0.0.1:11211'], + ); + + is_deeply($config->memcached_servers, ['127.0.0.1:11211'], "memcached_servers"); +} Index: branches/eam_branches/20090522/Nebulous-Server/t/02_server_setup.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/02_server_setup.t (revision 24529) +++ (revision ) @@ -1,65 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 02_server_setup.t,v 1.8 2008-12-14 22:54:25 eugene Exp $ - -use strict; -use warnings; - -use Test::More tests => 6; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -Test::Nebulous->setup; - -isa_ok(Nebulous::Server->new(), "Nebulous::Server"); - -Test::Nebulous->setup; - -# ->new() -{ - my $neb = Nebulous::Server->new( trace => 'off' ); - - ok($neb, "set log level"); -} - -Test::Nebulous->setup; - -{ - my $neb = Nebulous::Server->new( - dsn => "DBI:mysql:database=foobar:host=localhost", - dbuser => "baz", - dbpasswd =>"boo", - ); - - ok($neb, "set database"); -} - -Test::Nebulous->setup; - -# add dbs after ->new() -{ - my $neb = Nebulous::Server->new; - - ok($neb->config->add_db( - dbindex => 0, - dsn => "DBI:mysql:database=foobar:host=localhost", - dbuser => "baz", - dbpasswd =>"boo", - ), "add db"); - - ok($neb->config->add_db( - dbindex => 1, - dsn => "DBI:mysql:database=foobar:host=localhost", - dbuser => "baz", - dbpasswd =>"boo", - ), "add dbs"); - - is($neb->config->n_db, 2, "n dbs") -} - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/03_server_create_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/03_server_create_object.t (revision 24529) +++ (revision ) @@ -1,615 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 03_server_create_object.t,v 1.30 2008-09-11 22:35:52 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 99; - -use lib qw( ./t ./lib ); - -use File::Basename qw( basename ); -use File::ExtAttr qw( getfattr ); -use Nebulous::Server; -use Test::Nebulous; -use Test::URI; -use URI::Split qw( uri_split ); - -my $test_xattr = undef; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("neb:/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("neb://node01/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("neb://node02/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), '/foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; -{ - # key - my $uri = $neb->create_object("neb:///foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("/foo/"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), '/foo/', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("neb:/foo/"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), '/foo/', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("foo/"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo/', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("foo/bar"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo/bar"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo/bar', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("/foo/bar"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo/bar"), 'object key exists'); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), '/foo/bar', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key, volume - my $uri = $neb->create_object("foo", "node01"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # volume name override - my $uri = $neb->create_object("neb://node02/foo", "node01"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # volume name override - # OK because the volume arg overrides the key's implied volume - my $uri = $neb->create_object("neb://99/foo", "node01"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # undef volume name - # OK because the undef is ignored - my $uri = $neb->create_object("neb://node01/foo", undef); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # soft volume name request - my $uri = $neb->create_object("neb://~node01/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # soft volume name request - my $uri = $neb->create_object("neb://node01/foo", "~node02"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # any volume name request - my $uri = $neb->create_object("neb://any/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # any volume name request - my $uri = $neb->create_object("neb://~any/foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # any volume name request - my $uri = $neb->create_object("neb://node01/foo", "any"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # any volume name request - my $uri = $neb->create_object("neb://node01/foo", "~any"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - ok($neb->find_instances("foo"), 'object key exists'); - uri_scheme_ok($uri, 'file'); -} - -# test for properly row creation in the directories table - -Test::Nebulous->setup; - -{ - my $key = "foo"; - $neb->create_object($key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key, dir_id => 1], - ); -} - -Test::Nebulous->setup; - -{ - my $key = "a/foo"; - $neb->create_object($key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2], - ); - -} - -Test::Nebulous->setup; - -{ - my $key = "a/b/foo"; - $neb->create_object($key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 3], - ); -} - -Test::Nebulous->setup; - -{ - my $key = "a/b/c/foo"; - $neb->create_object($key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 4], - ); -} - -Test::Nebulous->setup; - -{ - my $key1 = "a/b/c/foo"; - my $key2 = "foo"; - $neb->create_object($key1); - $neb->create_object($key2); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], - ); -} - -Test::Nebulous->setup; - -{ - my $key1 = "a/b/c/foo"; - my $key2 = "a/foo"; - $neb->create_object($key1); - $neb->create_object($key2); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], - ); -} - -Test::Nebulous->setup; - -{ - my $key1 = "a/b/c/foo"; - my $key2 = "d/foo"; - $neb->create_object($key1); - $neb->create_object($key2); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], - directory => [dir_id => 5, dirname => 'd', parent_id => 1], - storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5], - ); -} - -Test::Nebulous->setup; - -{ - my $key1 = "a/b/c/foo"; - my $key2 = "a/d/foo"; - $neb->create_object($key1); - $neb->create_object($key2); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], - directory => [dir_id => 5, dirname => 'd', parent_id => 2], - storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5], - ); -} - -Test::Nebulous->setup; - -{ - my $key1 = "a/b/c/foo"; - my $key2 = "d/a/foo"; - $neb->create_object($key1); - $neb->create_object($key2); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], - directory => [dir_id => 5, dirname => 'd', parent_id => 1], - directory => [dir_id => 6, dirname => 'a', parent_id => 5], - storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 6], - ); -} - -Test::Nebulous->setup; - -{ - my $key1 = "a/b/c/foo"; - my $key2 = "a/b/c/d/foo"; - $neb->create_object($key1); - $neb->create_object($key2); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - directory => [dir_id => 3, dirname => 'b', parent_id => 2], - directory => [dir_id => 4, dirname => 'c', parent_id => 3], - storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], - directory => [dir_id => 5, dirname => 'd', parent_id => 4], - storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5], - ); -} - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - $neb->create_object("foo"); -}; -like($@, qr/Duplicate entry/, "object already exists"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("neb:/foo"); - $neb->create_object("neb:/foo"); -}; -like($@, qr/Duplicate entry/, "object already exists"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo", '~node07'); -}; -like($@, qr/node07 is not available/, "request volume this is full"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("neb://~node07/foo"); -}; -like($@, qr/node07 is not available/, "request volume this is full"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo", '~node04'); -}; -like($@, qr/node04 is not available/, "request volume with allocate = FALSE, available = FALSE"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("neb://~node04/foo"); -}; -like($@, qr/node04 is not available/, "request volume with allocate = FALSE, available = FALSE"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo", '~node05'); -}; -like($@, qr/node05 is not available/, "request volume with allocate = FALSE , available = TRUE"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("neb://~node05/foo"); -}; -like($@, qr/node05 is not available/, "request volume with allocate = FALSE , available = TRUE"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo", '~node06'); -}; -like($@, qr/node06 is not available/, "request volume with allocate = TRUE, available = FALSE"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("neb://~node06/foo"); -}; -like($@, qr/node06 is not available/, "request volume with allocate = TRUE, available = FALSE"); - -Test::Nebulous->setup; - -{ - ok($neb->create_object("foo", 99)); -} - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo", "~99"); -}; -like($@, qr/is not a valid volume name/, "volume name doesn't exist"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("neb://~99/foo"); -}; -like($@, qr/is not a valid volume name/, "volume name doesn't exist"); - -Test::Nebulous->setup; - -eval { - $neb->create_object(); -}; -like($@, qr/1 - 2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object(1, "node01", 3); -}; -like($@, qr/1 - 2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/03_server_setup.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/03_server_setup.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/03_server_setup.t (revision 24557) @@ -0,0 +1,71 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 02_server_setup.t,v 1.8 2008-12-14 22:54:25 eugene Exp $ + +use strict; +use warnings; + +use Test::More tests => 6; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +Test::Nebulous->setup; + +# ->new() + +{ + my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, + ); + + ok($neb, "set database"); +} + +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, + trace => 'off', + ); + + ok($neb, "set log level"); +} + +Test::Nebulous->setup; + +# add dbs after ->new() +{ + my $config = Nebulous::Server::Config->new; + + ok($config->add_db( + dbindex => 0, + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, + ), "add db"); + + ok($config->add_db( + dbindex => 1, + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, + ), "add dbs"); + + is($config->n_db, 2, "n dbs"); + + my $neb = Nebulous::Server->new_from_config($config); + + isa_ok($neb, 'Nebulous::Server'); +} + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/04_server_create_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/04_server_create_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/04_server_create_object.t (revision 24557) @@ -0,0 +1,615 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 03_server_create_object.t,v 1.30 2008-09-11 22:35:52 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 99; + +use lib qw( ./t ./lib ); + +use File::Basename qw( basename ); +use File::ExtAttr qw( getfattr ); +use Nebulous::Server; +use Test::Nebulous; +use Test::URI; +use URI::Split qw( uri_split ); + +my $test_xattr = undef; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("neb:/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("neb://node01/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("neb://node02/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), '/foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; +{ + # key + my $uri = $neb->create_object("neb:///foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("/foo/"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), '/foo/', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("neb:/foo/"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), '/foo/', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("foo/"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo/', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("foo/bar"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo/bar"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo/bar', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("/foo/bar"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo/bar"), 'object key exists'); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), '/foo/bar', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key, volume + my $uri = $neb->create_object("foo", "node01"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # volume name override + my $uri = $neb->create_object("neb://node02/foo", "node01"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # volume name override + # OK because the volume arg overrides the key's implied volume + my $uri = $neb->create_object("neb://99/foo", "node01"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # undef volume name + # OK because the undef is ignored + my $uri = $neb->create_object("neb://node01/foo", undef); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # soft volume name request + my $uri = $neb->create_object("neb://~node01/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # soft volume name request + my $uri = $neb->create_object("neb://node01/foo", "~node02"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # any volume name request + my $uri = $neb->create_object("neb://any/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # any volume name request + my $uri = $neb->create_object("neb://~any/foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # any volume name request + my $uri = $neb->create_object("neb://node01/foo", "any"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # any volume name request + my $uri = $neb->create_object("neb://node01/foo", "~any"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + ok($neb->find_instances("foo"), 'object key exists'); + uri_scheme_ok($uri, 'file'); +} + +# test for properly row creation in the directories table + +Test::Nebulous->setup; + +{ + my $key = "foo"; + $neb->create_object($key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key, dir_id => 1], + ); +} + +Test::Nebulous->setup; + +{ + my $key = "a/foo"; + $neb->create_object($key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2], + ); + +} + +Test::Nebulous->setup; + +{ + my $key = "a/b/foo"; + $neb->create_object($key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 3], + ); +} + +Test::Nebulous->setup; + +{ + my $key = "a/b/c/foo"; + $neb->create_object($key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 4], + ); +} + +Test::Nebulous->setup; + +{ + my $key1 = "a/b/c/foo"; + my $key2 = "foo"; + $neb->create_object($key1); + $neb->create_object($key2); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], + ); +} + +Test::Nebulous->setup; + +{ + my $key1 = "a/b/c/foo"; + my $key2 = "a/foo"; + $neb->create_object($key1); + $neb->create_object($key2); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], + ); +} + +Test::Nebulous->setup; + +{ + my $key1 = "a/b/c/foo"; + my $key2 = "d/foo"; + $neb->create_object($key1); + $neb->create_object($key2); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], + directory => [dir_id => 5, dirname => 'd', parent_id => 1], + storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5], + ); +} + +Test::Nebulous->setup; + +{ + my $key1 = "a/b/c/foo"; + my $key2 = "a/d/foo"; + $neb->create_object($key1); + $neb->create_object($key2); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], + directory => [dir_id => 5, dirname => 'd', parent_id => 2], + storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5], + ); +} + +Test::Nebulous->setup; + +{ + my $key1 = "a/b/c/foo"; + my $key2 = "d/a/foo"; + $neb->create_object($key1); + $neb->create_object($key2); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], + directory => [dir_id => 5, dirname => 'd', parent_id => 1], + directory => [dir_id => 6, dirname => 'a', parent_id => 5], + storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 6], + ); +} + +Test::Nebulous->setup; + +{ + my $key1 = "a/b/c/foo"; + my $key2 = "a/b/c/d/foo"; + $neb->create_object($key1); + $neb->create_object($key2); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + directory => [dir_id => 3, dirname => 'b', parent_id => 2], + directory => [dir_id => 4, dirname => 'c', parent_id => 3], + storage_object => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4], + directory => [dir_id => 5, dirname => 'd', parent_id => 4], + storage_object => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5], + ); +} + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + $neb->create_object("foo"); +}; +like($@, qr/Duplicate entry/, "object already exists"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("neb:/foo"); + $neb->create_object("neb:/foo"); +}; +like($@, qr/Duplicate entry/, "object already exists"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo", '~node07'); +}; +like($@, qr/node07 is not available/, "request volume this is full"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("neb://~node07/foo"); +}; +like($@, qr/node07 is not available/, "request volume this is full"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo", '~node04'); +}; +like($@, qr/node04 is not available/, "request volume with allocate = FALSE, available = FALSE"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("neb://~node04/foo"); +}; +like($@, qr/node04 is not available/, "request volume with allocate = FALSE, available = FALSE"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo", '~node05'); +}; +like($@, qr/node05 is not available/, "request volume with allocate = FALSE , available = TRUE"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("neb://~node05/foo"); +}; +like($@, qr/node05 is not available/, "request volume with allocate = FALSE , available = TRUE"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo", '~node06'); +}; +like($@, qr/node06 is not available/, "request volume with allocate = TRUE, available = FALSE"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("neb://~node06/foo"); +}; +like($@, qr/node06 is not available/, "request volume with allocate = TRUE, available = FALSE"); + +Test::Nebulous->setup; + +{ + ok($neb->create_object("foo", 99)); +} + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo", "~99"); +}; +like($@, qr/is not a valid volume name/, "volume name doesn't exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("neb://~99/foo"); +}; +like($@, qr/is not a valid volume name/, "volume name doesn't exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object(); +}; +like($@, qr/1 - 2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object(1, "node01", 3); +}; +like($@, qr/1 - 2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/04_server_replicate_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/04_server_replicate_object.t (revision 24529) +++ (revision ) @@ -1,185 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 04_server_replicate_object.t,v 1.16 2008-09-11 22:35:52 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 26; - -use lib qw( ./t ./lib ); - -use File::ExtAttr qw( getfattr ); -use Nebulous::Server; -use Test::Nebulous; -use Test::URI; -use URI::Split qw( uri_split ); - -my $test_xattr = undef; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - # key - $neb->create_object("foo"); - my $uri = $neb->replicate_object("foo"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key, $volume - $neb->create_object("foo"); - my $uri = $neb->replicate_object("foo", "node01"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key, $volume is undef - $neb->create_object("foo"); - my $uri = $neb->replicate_object("foo", undef); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "file exists"); - uri_scheme_ok($uri, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} -} - -Test::Nebulous->setup; - -{ - # key, $volume - $neb->create_object("foo"); - my $uri1 = $neb->replicate_object("foo", "~node01"); - my $uri2 = $neb->replicate_object("foo", "~node02"); - - { - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri1); - ok(-e $path, "file exists"); - uri_scheme_ok($uri1, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} - } - - { - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri2); - ok(-e $path, "file exists"); - uri_scheme_ok($uri2, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} - } -} - -Test::Nebulous->setup; - -{ - # key, $volume - $neb->create_object("foo"); - my $uri1 = $neb->replicate_object("foo", "any"); - - { - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri1); - ok(-e $path, "file exists"); - uri_scheme_ok($uri1, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} - } -} - -Test::Nebulous->setup; - -{ - # key, $volume - $neb->create_object("foo"); - my $uri1 = $neb->replicate_object("foo", "~any"); - - { - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri1); - ok(-e $path, "file exists"); - uri_scheme_ok($uri1, 'file'); - -SKIP: { - skip "requires xattr support", 1 unless $test_xattr; - is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); -} - } -} - -Test::Nebulous->setup; - -eval { - $neb->replicate_object('foo'); -}; -like($@, qr/is valid object key/, 'storage object does not exist'); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - ok($neb->replicate_object('foo', 'bar'),'soft fake storage volume'); -}; - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - $neb->replicate_object('foo', '~bar'); -}; -like($@, qr/is not a valid volume name/, 'storage volume does not exist'); - -Test::Nebulous->setup; - -eval { - $neb->replicate_object(); -}; -like($@, qr/1 - 2 were expected/, 'no params'); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - $neb->replicate_object('foo', 'node01', 3); -}; -like($@, qr/1 - 2 were expected/, 'too many params'); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/05_server_lock_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/05_server_lock_object.t (revision 24529) +++ (revision ) @@ -1,126 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 05_server_lock_object.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 13; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - ok($neb->lock_object("foo", "read"), "read lock"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - ok($neb->lock_object("foo", "read"), "read lock"); - ok($neb->lock_object("foo", "read"), "read lock"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - ok($neb->lock_object("foo", "write"), "write lock"); -} - -Test::Nebulous->setup; - -eval { - $neb->lock_object("foo", "read"); -}; -like($@, qr/is valid object key/, "storage object does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->lock_object("foo", "write"); -}; -like($@, qr/is valid object key/, "storage object does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->lock_object("foo", "write"); - $neb->lock_object("foo", "write"); -}; -like($@, qr/can not write lock twice/, "can not write lock twice"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->lock_object("foo", "read"); - $neb->lock_object("foo", "write"); -}; -like($@, qr/can not write lock after read lock/, "can not write lock after read lock"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->lock_object("foo", "write"); - $neb->lock_object("foo", "read"); -}; -like($@, qr/can not read lock after write lock/, "can not read lock after write lock"); - -Test::Nebulous->setup; - -eval { - $neb->lock_object(); -}; -like($@, qr/2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->lock_object("foo"); -}; -like($@, qr/2 were expected/, "not enough params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->lock_object("foo", "both"); -}; -like($@, qr/is read or write/, "not read or write"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->lock_object("foo", 'read', 3); -}; -like($@, qr/2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/05_server_replicate_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/05_server_replicate_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/05_server_replicate_object.t (revision 24557) @@ -0,0 +1,186 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 04_server_replicate_object.t,v 1.16 2008-09-11 22:35:52 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 26; + +use lib qw( ./t ./lib ); + +use File::ExtAttr qw( getfattr ); +use Nebulous::Server; +use Test::Nebulous; +use Test::URI; +use URI::Split qw( uri_split ); + +my $test_xattr = undef; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, + memcached_servers => $NEB_MEMCACHED_SERVERS, +); + +Test::Nebulous->setup; + +{ + # key + $neb->create_object("foo"); + my $uri = $neb->replicate_object("foo"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key, $volume + $neb->create_object("foo"); + my $uri = $neb->replicate_object("foo", "node01"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key, $volume is undef + $neb->create_object("foo"); + my $uri = $neb->replicate_object("foo", undef); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "file exists"); + uri_scheme_ok($uri, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} +} + +Test::Nebulous->setup; + +{ + # key, $volume + $neb->create_object("foo"); + my $uri1 = $neb->replicate_object("foo", "~node01"); + my $uri2 = $neb->replicate_object("foo", "~node02"); + + { + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri1); + ok(-e $path, "file exists"); + uri_scheme_ok($uri1, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} + } + + { + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri2); + ok(-e $path, "file exists"); + uri_scheme_ok($uri2, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} + } +} + +Test::Nebulous->setup; + +{ + # key, $volume + $neb->create_object("foo"); + my $uri1 = $neb->replicate_object("foo", "any"); + + { + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri1); + ok(-e $path, "file exists"); + uri_scheme_ok($uri1, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} + } +} + +Test::Nebulous->setup; + +{ + # key, $volume + $neb->create_object("foo"); + my $uri1 = $neb->replicate_object("foo", "~any"); + + { + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri1); + ok(-e $path, "file exists"); + uri_scheme_ok($uri1, 'file'); + +SKIP: { + skip "requires xattr support", 1 unless $test_xattr; + is(getfattr($path, 'user.nebulous_key'), 'foo', 'user.nebulous_key xattr'); +} + } +} + +Test::Nebulous->setup; + +eval { + $neb->replicate_object('foo'); +}; +like($@, qr/is valid object key/, 'storage object does not exist'); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + ok($neb->replicate_object('foo', 'bar'),'soft fake storage volume'); +}; + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + $neb->replicate_object('foo', '~bar'); +}; +like($@, qr/is not a valid volume name/, 'storage volume does not exist'); + +Test::Nebulous->setup; + +eval { + $neb->replicate_object(); +}; +like($@, qr/1 - 2 were expected/, 'no params'); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + $neb->replicate_object('foo', 'node01', 3); +}; +like($@, qr/1 - 2 were expected/, 'too many params'); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/06_server_lock_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/06_server_lock_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/06_server_lock_object.t (revision 24557) @@ -0,0 +1,126 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 05_server_lock_object.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 13; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + ok($neb->lock_object("foo", "read"), "read lock"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + ok($neb->lock_object("foo", "read"), "read lock"); + ok($neb->lock_object("foo", "read"), "read lock"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + ok($neb->lock_object("foo", "write"), "write lock"); +} + +Test::Nebulous->setup; + +eval { + $neb->lock_object("foo", "read"); +}; +like($@, qr/is valid object key/, "storage object does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->lock_object("foo", "write"); +}; +like($@, qr/is valid object key/, "storage object does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->lock_object("foo", "write"); + $neb->lock_object("foo", "write"); +}; +like($@, qr/can not write lock twice/, "can not write lock twice"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->lock_object("foo", "read"); + $neb->lock_object("foo", "write"); +}; +like($@, qr/can not write lock after read lock/, "can not write lock after read lock"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->lock_object("foo", "write"); + $neb->lock_object("foo", "read"); +}; +like($@, qr/can not read lock after write lock/, "can not read lock after write lock"); + +Test::Nebulous->setup; + +eval { + $neb->lock_object(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->lock_object("foo"); +}; +like($@, qr/2 were expected/, "not enough params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->lock_object("foo", "both"); +}; +like($@, qr/is read or write/, "not read or write"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->lock_object("foo", 'read', 3); +}; +like($@, qr/2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/06_server_unlock_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/06_server_unlock_object.t (revision 24529) +++ (revision ) @@ -1,141 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 06_server_unlock_object.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 14; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - $neb->lock_object("foo", "read"); - - ok($neb->unlock_object("foo", "read"), "read unlock"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - $neb->lock_object("foo", "read"); - $neb->lock_object("foo", "read"); - - ok($neb->unlock_object("foo", "read"), "read unlock"); - ok($neb->unlock_object("foo", "read"), "read unlock"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - $neb->lock_object("foo", "write"); - - ok($neb->unlock_object("foo", "write"), "write unlock"); -} - -Test::Nebulous->setup; - -eval { - $neb->unlock_object("foo", "read"); -}; -like($@, qr/is valid object key/, "storage object does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->unlock_object("foo", "write"); -}; -like($@, qr/is valid object key/, "storage object does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->unlock_object("foo", "read"); -}; -like($@, qr/can not remove non-existant read lock/, "no lock set"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->unlock_object("foo", "write"); -}; -like($@, qr/can not remove non-existant write lock/, "no lock set"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - $neb->lock_object("foo", "write"); - - $neb->unlock_object("foo", "read"); -}; -like($@, qr/can not have a read lock under a write lock/, "read unlock under write lock"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - $neb->lock_object("foo", "read"); - - $neb->unlock_object("foo", "write"); -}; -like($@, qr/can not have a write lock under a read lock/, "write unlock under read lock"); - -Test::Nebulous->setup; - -eval { - $neb->unlock_object(); -}; -like($@, qr/2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->unlock_object("foo"); -}; -like($@, qr/2 were expected/, "not enough params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->unlock_object("foo", "both"); -}; -like($@, qr/is read or write/, "not read or write"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->unlock_object("foo", 'read', 3); -}; -like($@, qr/2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/07_server_find_instances.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/07_server_find_instances.t (revision 24529) +++ (revision ) @@ -1,142 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 07_server_find_instances.t,v 1.16 2008-09-11 22:35:52 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 19; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::URI; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - # key - my $uri = $neb->create_object("foo"); - - my $locations = $neb->find_instances("foo"); - - uri_scheme_ok($locations->[0], 'file'); - is($uri, $locations->[0], "URIs match"); -} - -Test::Nebulous->setup; - -{ - # key - my $uri1 = $neb->create_object("foo"); - my $uri2 = $neb->replicate_object("foo"); - - my $locations = $neb->find_instances("foo"); - - uri_scheme_ok($locations->[0], 'file'); - uri_scheme_ok($locations->[1], 'file'); - ok(eq_set([$uri1, $uri2], $locations), "URIs match"); -} - -Test::Nebulous->setup; - -{ - # key, volume - my $uri = $neb->create_object('foo', 'node01'); - - my $locations = $neb->find_instances('foo', 'node01'); - - uri_scheme_ok($locations->[0], 'file'); - is($uri, $locations->[0], "URIs match"); -} - -Test::Nebulous->setup; - -{ - # key, volume - my $uri = $neb->create_object('foo', 'node01'); - - my $locations = $neb->find_instances('foo', undef); - - uri_scheme_ok($locations->[0], 'file'); - is($uri, $locations->[0], "URIs match"); -} - -Test::Nebulous->setup; - -{ - # key, volume - my $uri = $neb->create_object('foo', 'node01'); - - my $locations = $neb->find_instances('neb://node02/foo', "~any"); - - uri_scheme_ok($locations->[0], 'file'); - is($uri, $locations->[0], "URIs match"); -} - -Test::Nebulous->setup; - -{ - # key - my $uri1 = $neb->create_object("foo"); - my $uri2 = $neb->replicate_object("foo"); - - my $locations = $neb->find_instances("foo", "~any"); - - uri_scheme_ok($locations->[0], 'file'); - uri_scheme_ok($locations->[1], 'file'); - ok(eq_set([$uri1, $uri2], $locations), "URIs match"); -} - -# object exists but instance is on a different volume -Test::Nebulous->setup; - -eval { - # key, volume - my $uri = $neb->create_object('foo', '~node01'); - - my $locations = $neb->find_instances('foo', '~node02'); -}; -like($@, qr/no instances on storage volume/, 'instances on a different volume'); - - -Test::Nebulous->setup; - -eval { - $neb->find_instances('foo'); -}; -like($@, qr/is valid object key/, "storage object does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - $neb->find_instances('foo', '~bar'); -}; -like($@, qr/is not a valid volume name/, "storage volume does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->find_instances(); -}; -like($@, qr/1 - 2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - $neb->find_instances('foo', 'node01', 3); -}; -like($@, qr/1 - 2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/07_server_unlock_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/07_server_unlock_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/07_server_unlock_object.t (revision 24557) @@ -0,0 +1,141 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 06_server_unlock_object.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 14; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + $neb->lock_object("foo", "read"); + + ok($neb->unlock_object("foo", "read"), "read unlock"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + $neb->lock_object("foo", "read"); + $neb->lock_object("foo", "read"); + + ok($neb->unlock_object("foo", "read"), "read unlock"); + ok($neb->unlock_object("foo", "read"), "read unlock"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + $neb->lock_object("foo", "write"); + + ok($neb->unlock_object("foo", "write"), "write unlock"); +} + +Test::Nebulous->setup; + +eval { + $neb->unlock_object("foo", "read"); +}; +like($@, qr/is valid object key/, "storage object does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->unlock_object("foo", "write"); +}; +like($@, qr/is valid object key/, "storage object does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->unlock_object("foo", "read"); +}; +like($@, qr/can not remove non-existant read lock/, "no lock set"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->unlock_object("foo", "write"); +}; +like($@, qr/can not remove non-existant write lock/, "no lock set"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + $neb->lock_object("foo", "write"); + + $neb->unlock_object("foo", "read"); +}; +like($@, qr/can not have a read lock under a write lock/, "read unlock under write lock"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + $neb->lock_object("foo", "read"); + + $neb->unlock_object("foo", "write"); +}; +like($@, qr/can not have a write lock under a read lock/, "write unlock under read lock"); + +Test::Nebulous->setup; + +eval { + $neb->unlock_object(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->unlock_object("foo"); +}; +like($@, qr/2 were expected/, "not enough params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->unlock_object("foo", "both"); +}; +like($@, qr/is read or write/, "not read or write"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->unlock_object("foo", 'read', 3); +}; +like($@, qr/2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/08_server_delete_instance.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/08_server_delete_instance.t (revision 24529) +++ (revision ) @@ -1,80 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 08_server_delete_instance.t,v 1.10.22.1 2008-12-14 22:52:37 eugene Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 8; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - my $key = "foo"; - my $uri = $neb->create_object($key); - - ok($neb->delete_instance($key, $uri), "delete instance"); -} - -Test::Nebulous->setup; - -{ - my $key = "foo"; - my $uri1 = $neb->create_object($key); - my $uri2 = $neb->replicate_object($key); - - ok($neb->delete_instance($key, $uri1), "delete instance"); - - my $locations = $neb->find_instances($key); - - is($locations->[0], $uri2, "instance remains"); - - ok($neb->delete_instance($key, $uri2), "delete instance"); - - eval { - $neb->find_instances($key); - }; - like($@, qr/is valid object key/, "storage object was deleted"); -} - -Test::Nebulous->setup; - -eval { - my $key = "foo"; - my $uri1 = $neb->create_object($key); - - $neb->delete_instance($key, "file:/foo"); -}; -like($@, qr/no instance is associated with uri/, "uri does not exist"); - -Test::Nebulous->setup; - -eval { - $neb->delete_instance(); -}; -like($@, qr/2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - my $key = "foo"; - my $uri1 = $neb->create_object($key); - - $neb->delete_instance("foo", 2, 3); -}; -like($@, qr/2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/08_server_find_instances.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/08_server_find_instances.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/08_server_find_instances.t (revision 24557) @@ -0,0 +1,141 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 07_server_find_instances.t,v 1.16 2008-09-11 22:35:52 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 19; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::URI; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +{ + # key + my $uri = $neb->create_object("foo"); + + my $locations = $neb->find_instances("foo"); + + uri_scheme_ok($locations->[0], 'file'); + is($uri, $locations->[0], "URIs match"); +} + +Test::Nebulous->setup; + +{ + # key + my $uri1 = $neb->create_object("foo"); + my $uri2 = $neb->replicate_object("foo"); + + my $locations = $neb->find_instances("foo"); + + uri_scheme_ok($locations->[0], 'file'); + uri_scheme_ok($locations->[1], 'file'); + ok(eq_set([$uri1, $uri2], $locations), "URIs match"); +} + +Test::Nebulous->setup; + +{ + # key, volume + my $uri = $neb->create_object('foo', 'node01'); + + my $locations = $neb->find_instances('foo', 'node01'); + + uri_scheme_ok($locations->[0], 'file'); + is($uri, $locations->[0], "URIs match"); +} + +Test::Nebulous->setup; + +{ + # key, volume + my $uri = $neb->create_object('foo', 'node01'); + + my $locations = $neb->find_instances('foo', undef); + + uri_scheme_ok($locations->[0], 'file'); + is($uri, $locations->[0], "URIs match"); +} + +Test::Nebulous->setup; + +{ + # key, volume + my $uri = $neb->create_object('foo', 'node01'); + + my $locations = $neb->find_instances('neb://node02/foo', "~any"); + + uri_scheme_ok($locations->[0], 'file'); + is($uri, $locations->[0], "URIs match"); +} + +Test::Nebulous->setup; + +{ + # key + my $uri1 = $neb->create_object("foo"); + my $uri2 = $neb->replicate_object("foo"); + + my $locations = $neb->find_instances("foo", "~any"); + + uri_scheme_ok($locations->[0], 'file'); + uri_scheme_ok($locations->[1], 'file'); + ok(eq_set([$uri1, $uri2], $locations), "URIs match"); +} + +# object exists but instance is on a different volume +Test::Nebulous->setup; + +eval { + # key, volume + my $uri = $neb->create_object('foo', '~node01'); + + my $locations = $neb->find_instances('foo', '~node02'); +}; +like($@, qr/no instances on storage volume/, 'instances on a different volume'); + +Test::Nebulous->setup; + +eval { + $neb->find_instances('foo'); +}; +like($@, qr/is valid object key/, "storage object does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + $neb->find_instances('foo', '~bar'); +}; +like($@, qr/is not a valid volume name/, "storage volume does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->find_instances(); +}; +like($@, qr/1 - 2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + $neb->find_instances('foo', 'node01', 3); +}; +like($@, qr/1 - 2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/09_server_delete_instance.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/09_server_delete_instance.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/09_server_delete_instance.t (revision 24557) @@ -0,0 +1,80 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 08_server_delete_instance.t,v 1.10.22.1 2008-12-14 22:52:37 eugene Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 8; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $uri = $neb->create_object($key); + + ok($neb->delete_instance($key, $uri), "delete instance"); +} + +Test::Nebulous->setup; + +{ + my $key = "foo"; + my $uri1 = $neb->create_object($key); + my $uri2 = $neb->replicate_object($key); + + ok($neb->delete_instance($key, $uri1), "delete instance"); + + my $locations = $neb->find_instances($key); + + is($locations->[0], $uri2, "instance remains"); + + ok($neb->delete_instance($key, $uri2), "delete instance"); + + eval { + $neb->find_instances($key); + }; + like($@, qr/is valid object key/, "storage object was deleted"); +} + +Test::Nebulous->setup; + +eval { + my $key = "foo"; + my $uri1 = $neb->create_object($key); + + $neb->delete_instance($key, "file:/foo"); +}; +like($@, qr/no instance is associated with uri/, "uri does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->delete_instance(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + my $key = "foo"; + my $uri1 = $neb->create_object($key); + + $neb->delete_instance("foo", 2, 3); +}; +like($@, qr/2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/09_server_stat_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/09_server_stat_object.t (revision 24529) +++ (revision ) @@ -1,71 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 09_server_stat_object.t,v 1.15 2008-05-16 20:29:19 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 12; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; -{ - $neb->create_object("foo"); - - my $info = $neb->stat_object("foo"); - - is(scalar @$info, 7, "number of columns"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("foo", "node01"); - - my $info = $neb->stat_object("foo"); - - is(scalar @$info, 7, "number of columns"); - is(@$info[0], 1, "so_id"); - is(@$info[1], "foo", "ext_id"); - is(@$info[2], 0, "read lock"); - is(@$info[3], undef, "write lock"); - like(@$info[4], qr/....-..-.. ..:..:../, "epoch"); - like(@$info[5], qr/....-..-.. ..:..:../, "mtime"); - is(@$info[6], 1, "instances"); -} - -Test::Nebulous->setup; - -eval { - my $stat = $neb->stat_object("foo"); -}; -like($@, qr/is valid object key/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->stat_object(); -}; -like($@, qr/1 was expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - $neb->stat_object("foo", 2); -}; -like($@, qr/1 was expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/10_server_is_valid_volume_name.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/10_server_is_valid_volume_name.t (revision 24529) +++ (revision ) @@ -1,35 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 10_server_is_valid_volume_name.t,v 1.7 2008-12-14 22:54:25 eugene Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 3; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -ok($neb->_is_valid_volume_name('foo', 'node01'), "valid volume name"); - -Test::Nebulous->setup; - -ok($neb->_is_valid_volume_name('foo', 'node02'), "valid volume name"); - -Test::Nebulous->setup; - -is($neb->_is_valid_volume_name('foo', 'node99'), undef, "invalid volume name"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/10_server_stat_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/10_server_stat_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/10_server_stat_object.t (revision 24557) @@ -0,0 +1,71 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 09_server_stat_object.t,v 1.15 2008-05-16 20:29:19 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 12; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; +{ + $neb->create_object("foo"); + + my $info = $neb->stat_object("foo"); + + is(scalar @$info, 7, "number of columns"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("foo", "node01"); + + my $info = $neb->stat_object("foo"); + + is(scalar @$info, 7, "number of columns"); + is(@$info[0], 1, "so_id"); + is(@$info[1], "foo", "ext_id"); + is(@$info[2], 0, "read lock"); + is(@$info[3], undef, "write lock"); + like(@$info[4], qr/....-..-.. ..:..:../, "epoch"); + like(@$info[5], qr/....-..-.. ..:..:../, "mtime"); + is(@$info[6], 1, "instances"); +} + +Test::Nebulous->setup; + +eval { + my $stat = $neb->stat_object("foo"); +}; +like($@, qr/is valid object key/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->stat_object(); +}; +like($@, qr/1 was expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + $neb->stat_object("foo", 2); +}; +like($@, qr/1 was expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/11_server_is_valid_object_key.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/11_server_is_valid_object_key.t (revision 24529) +++ (revision ) @@ -1,37 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 11_server_is_valid_object_key.t,v 1.3 2008-03-20 21:10:14 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 2; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - $neb->create_object('foo'); - - ok($neb->_is_valid_object_key('foo'), "valid object key"); -} - -Test::Nebulous->setup; - -{ - ok(!$neb->_is_valid_object_key('foo'), "invalid object key"); -} - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/11_server_is_valid_volume_name.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/11_server_is_valid_volume_name.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/11_server_is_valid_volume_name.t (revision 24557) @@ -0,0 +1,35 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 10_server_is_valid_volume_name.t,v 1.7 2008-12-14 22:54:25 eugene Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 3; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +ok($neb->_is_valid_volume_name('foo', 'node01'), "valid volume name"); + +Test::Nebulous->setup; + +ok($neb->_is_valid_volume_name('foo', 'node02'), "valid volume name"); + +Test::Nebulous->setup; + +is($neb->_is_valid_volume_name('foo', 'node99'), undef, "invalid volume name"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/12_server_find_objects.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/12_server_find_objects.t (revision 24529) +++ (revision ) @@ -1,180 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 12_server_find_objects.t,v 1.4 2008-05-16 20:29:19 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 28; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -# search for a regex of '' should match nothing -eval { - $neb->create_object("foo"); - - my $keys = $neb->find_objects(); -}; -like($@, qr/no keys found/, "no keys found"); - -Test::Nebulous->setup; - -{ - $neb->create_object("foo"); - - my $keys = $neb->find_objects("foo"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "foo", "key name"); -} - -Test::Nebulous->setup; - -{ - # key - $neb->create_object("foo"); - $neb->replicate_object("foo"); - - my $keys = $neb->find_objects("foo"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "foo", "key name"); -} - -Test::Nebulous->setup; - -{ - # key - $neb->create_object("foo"); - $neb->create_object("bar"); - - my $keys = $neb->find_objects("foo"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "foo", "key name"); -} - -# test recursive dir searching -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - - my $keys = $neb->find_objects("a"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "a/foo", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("b/foo"); - - my $keys = $neb->find_objects("a"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "a/foo", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("a/b/foo"); - - my $keys = $neb->find_objects("a"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "a/foo", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("a/bar"); - - my $keys = $neb->find_objects("a"); - - is(scalar @$keys, 2, 'number of keys found'); - is($keys->[0], "a/foo", "key name"); - is($keys->[1], "a/bar", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("bar"); - - my $keys = $neb->find_objects("a"); - - is(scalar @$keys, 1, 'number of keys found'); - is($keys->[0], "a/foo", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("foo"); - $neb->create_object("bar"); - - my $keys = $neb->find_objects("/"); - - is(scalar @$keys, 2, 'number of keys found'); - is($keys->[0], "foo", "key name"); - is($keys->[1], "bar", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("foo"); - $neb->create_object("bar"); - - my $keys = $neb->find_objects("."); - - is(scalar @$keys, 2, 'number of keys found'); - is($keys->[0], "foo", "key name"); - is($keys->[1], "bar", "key name"); -} - -Test::Nebulous->setup; - -{ - $neb->create_object("a/foo"); - $neb->create_object("foo"); - $neb->create_object("bar"); - - my $keys = $neb->find_objects(".."); - - is(scalar @$keys, 2, 'number of keys found'); - is($keys->[0], "foo", "key name"); - is($keys->[1], "bar", "key name"); -} - -Test::Nebulous->setup; - -eval { - $neb->find_objects("foo", 3); -}; -like($@, qr/1 was expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/12_server_is_valid_object_key.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/12_server_is_valid_object_key.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/12_server_is_valid_object_key.t (revision 24557) @@ -0,0 +1,37 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 11_server_is_valid_object_key.t,v 1.3 2008-03-20 21:10:14 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 2; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +{ + $neb->create_object('foo'); + + ok($neb->_is_valid_object_key('foo'), "valid object key"); +} + +Test::Nebulous->setup; + +{ + ok(!$neb->_is_valid_object_key('foo'), "invalid object key"); +} + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/13_server_find_objects.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/13_server_find_objects.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/13_server_find_objects.t (revision 24557) @@ -0,0 +1,180 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 12_server_find_objects.t,v 1.4 2008-05-16 20:29:19 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 28; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +# search for a regex of '' should match nothing +eval { + $neb->create_object("foo"); + + my $keys = $neb->find_objects(); +}; +like($@, qr/no keys found/, "no keys found"); + +Test::Nebulous->setup; + +{ + $neb->create_object("foo"); + + my $keys = $neb->find_objects("foo"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "foo", "key name"); +} + +Test::Nebulous->setup; + +{ + # key + $neb->create_object("foo"); + $neb->replicate_object("foo"); + + my $keys = $neb->find_objects("foo"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "foo", "key name"); +} + +Test::Nebulous->setup; + +{ + # key + $neb->create_object("foo"); + $neb->create_object("bar"); + + my $keys = $neb->find_objects("foo"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "foo", "key name"); +} + +# test recursive dir searching +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + + my $keys = $neb->find_objects("a"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "a/foo", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("b/foo"); + + my $keys = $neb->find_objects("a"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "a/foo", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("a/b/foo"); + + my $keys = $neb->find_objects("a"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "a/foo", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("a/bar"); + + my $keys = $neb->find_objects("a"); + + is(scalar @$keys, 2, 'number of keys found'); + is($keys->[0], "a/foo", "key name"); + is($keys->[1], "a/bar", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("bar"); + + my $keys = $neb->find_objects("a"); + + is(scalar @$keys, 1, 'number of keys found'); + is($keys->[0], "a/foo", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("foo"); + $neb->create_object("bar"); + + my $keys = $neb->find_objects("/"); + + is(scalar @$keys, 2, 'number of keys found'); + is($keys->[0], "foo", "key name"); + is($keys->[1], "bar", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("foo"); + $neb->create_object("bar"); + + my $keys = $neb->find_objects("."); + + is(scalar @$keys, 2, 'number of keys found'); + is($keys->[0], "foo", "key name"); + is($keys->[1], "bar", "key name"); +} + +Test::Nebulous->setup; + +{ + $neb->create_object("a/foo"); + $neb->create_object("foo"); + $neb->create_object("bar"); + + my $keys = $neb->find_objects(".."); + + is(scalar @$keys, 2, 'number of keys found'); + is($keys->[0], "foo", "key name"); + is($keys->[1], "bar", "key name"); +} + +Test::Nebulous->setup; + +eval { + $neb->find_objects("foo", 3); +}; +like($@, qr/1 was expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/13_server_rename_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/13_server_rename_object.t (revision 24529) +++ (revision ) @@ -1,122 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2007 Joshua Hoblitt -# -# $Id: 13_server_rename_object.t,v 1.5 2008-05-16 20:29:19 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 8; - -use lib qw( ./t ./lib ); - -use File::Basename qw( basename ); -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; - -Test::Nebulous->setup; - -{ - my $key = "bar"; - my $uri = $neb->create_object("foo"); - - $neb->rename_object("foo", $key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 1], - ); -} - -Test::Nebulous->setup; - -{ - my $key = "a/bar"; - my $uri = $neb->create_object("foo"); - - $neb->rename_object("foo", $key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2], - ); -} - -Test::Nebulous->setup; - -{ - my $key = "bar"; - my $uri = $neb->create_object("a/foo"); - - $neb->rename_object("a/foo", $key); - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 1], - ); -} - -Test::Nebulous->setup; - -# destination key exists -eval { - $neb->create_object('foo'); - $neb->create_object('bar'); - - $neb->rename_object('foo', 'bar'); -}; -like($@, qr/is not valid object key/, "too few params"); - -Test::Nebulous->setup; - -eval { - $neb->rename_object(); -}; -like($@, qr/2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->rename_object("foo"); -}; -like($@, qr/2 were expected/, "too few params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->rename_object("foo", "bar", "baz"); -}; -like($@, qr/2 were expected/, "too many params"); - -# test attempting to rename a key in such a way to cause the distributed -# storage db to change -# this must be the last test as we're messing with the $neb object -eval { - $neb->config->add_db( - dbindex => 1, - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, - ); - - $neb->create_object("a/foo"); - $neb->rename_object("a/foo", "g/bar"); -}; -like($@, qr/rename objects across distributed database boundaries/, "rename between databases"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/14_server_rename_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/14_server_rename_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/14_server_rename_object.t (revision 24557) @@ -0,0 +1,122 @@ +#!/usr/bin/perl + +# Copryight (C) 2007 Joshua Hoblitt +# +# $Id: 13_server_rename_object.t,v 1.5 2008-05-16 20:29:19 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 8; + +use lib qw( ./t ./lib ); + +use File::Basename qw( basename ); +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; + +Test::Nebulous->setup; + +{ + my $key = "bar"; + my $uri = $neb->create_object("foo"); + + $neb->rename_object("foo", $key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 1], + ); +} + +Test::Nebulous->setup; + +{ + my $key = "a/bar"; + my $uri = $neb->create_object("foo"); + + $neb->rename_object("foo", $key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2], + ); +} + +Test::Nebulous->setup; + +{ + my $key = "bar"; + my $uri = $neb->create_object("a/foo"); + + $neb->rename_object("a/foo", $key); + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 1], + ); +} + +Test::Nebulous->setup; + +# destination key exists +eval { + $neb->create_object('foo'); + $neb->create_object('bar'); + + $neb->rename_object('foo', 'bar'); +}; +like($@, qr/is not valid object key/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->rename_object(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->rename_object("foo"); +}; +like($@, qr/2 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->rename_object("foo", "bar", "baz"); +}; +like($@, qr/2 were expected/, "too many params"); + +# test attempting to rename a key in such a way to cause the distributed +# storage db to change +# this must be the last test as we're messing with the $neb object +eval { + $neb->config->add_db( + dbindex => 1, + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, + ); + + $neb->create_object("a/foo"); + $neb->rename_object("a/foo", "g/bar"); +}; +like($@, qr/rename objects across distributed database boundaries/, "rename between databases"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/14_server_xattr.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/14_server_xattr.t (revision 24529) +++ (revision ) @@ -1,214 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2007 Joshua Hoblitt -# -# $Id: 14_server_xattr.t,v 1.8 2008-07-09 23:32:35 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 36; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -# 1 key / xattr - -Test::Nebulous->setup; - -{ - my $uri = $neb->create_object('foo'); - - ok($neb->setxattr_object('foo', 'bar', 'baz', 'create'), 'set object xattr'); - { - my $xattrs = $neb->listxattr_object('foo'); - is(scalar @$xattrs, 1, 'number of xattrs'); - is(@$xattrs[0], 'bar', 'xattr name'); - } - - my $value = $neb->getxattr_object('foo', 'bar'); - is($value, 'baz', 'xattr value'); - - ok($neb->removexattr_object('foo', 'bar'), "remove object xattr"); - { - my $xattrs = $neb->listxattr_object('foo'); - is(scalar @$xattrs, 0, 'number of xattrs'); - } -} - -# multiple xattrs - -Test::Nebulous->setup; - -{ - my $uri = $neb->create_object('foo'); - - ok($neb->setxattr_object('foo', 'bar', 'baz', 'create'), 'set object xattr'); - ok($neb->setxattr_object('foo', 'bonk', 'quix', 'create'), 'set object xattr'); - - { - my $xattrs = $neb->listxattr_object('foo'); - is(scalar @$xattrs, 2, 'number of xattrs'); - is(@$xattrs[0], 'bar', 'xattr name'); - is(@$xattrs[1], 'bonk', 'xattr name'); - } - - my $value = $neb->getxattr_object('foo', 'bar'); - is($value, 'baz', 'xattr value'); - $value = $neb->getxattr_object('foo', 'bonk'); - is($value, 'quix', 'xattr value'); - - ok($neb->removexattr_object('foo', 'bar'), "remove object xattr"); - ok($neb->removexattr_object('foo', 'bonk'), "remove object xattr"); - { - my $xattrs = $neb->listxattr_object('foo'); - is(scalar @$xattrs, 0, 'number of xattrs'); - } -} - -# replace xattrs - -Test::Nebulous->setup; - -{ - my $uri = $neb->create_object('foo'); - - ok($neb->setxattr_object('foo', 'bar', 'baz', 'create'), 'set object xattr'); - ok($neb->setxattr_object('foo', 'bar', 'quix', 'replace'), 're-set object xattr'); - - { - my $xattrs = $neb->listxattr_object('foo'); - is(scalar @$xattrs, 1, 'number of xattrs'); - is(@$xattrs[0], 'bar', 'xattr name'); - } - - my $value = $neb->getxattr_object('foo', 'bar'); - is($value, 'quix', 'xattr value'); - - ok($neb->removexattr_object('foo', 'bar'), "remove object xattr"); - { - my $xattrs = $neb->listxattr_object('foo'); - is(scalar @$xattrs, 0, 'number of xattrs'); - } -} - -# setxattr_object - -Test::Nebulous->setup; - -eval { - $neb->setxattr_object('foo', 'bar', 'baz', 'create'); -}; -like($@, qr/is valid object key/, "create xattr on non-existant key"); - -eval { - $neb->setxattr_object('foo', 'bar', 'baz', 'replace'); -}; -like($@, qr/is valid object key/, "replace xattr on non-existant key"); - -Test::Nebulous->setup; - -eval { - $neb->setxattr_object(); -}; -like($@, qr/4 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->setxattr_object('foo', 'bar'); -}; -like($@, qr/4 were expected/, "too few params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->setxattr_object('foo', 'bar', 'baz'); -}; -like($@, qr/4 were expected/, "too few params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->setxattr_object('foo', 'bar', 'baz', 'create', 'quix'); -}; -like($@, qr/4 were expected/, "too many params"); - -# getxattr_object - -Test::Nebulous->setup; - -eval { - $neb->getxattr_object('foo', 'bar'); -}; -like($@, qr/is valid object key/, "get xattr from non-existant nebulous key"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - $neb->getxattr_object('foo', 'bar'); -}; -like($@, qr|xattr neb:///foo:bar does not exist|, - "get xattr from non-existant xattr key"); - -Test::Nebulous->setup; - -eval { - $neb->getxattr_object(); -}; -like($@, qr/2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->getxattr_object('foo'); -}; -like($@, qr/2 were expected/, "too few params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->getxattr_object('foo', 'bar', 'baz'); -}; -like($@, qr/2 were expected/, "too many params"); - -# listxattr_object - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->listxattr_object(); -}; -like($@, qr/1 was expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->listxattr_object('foo', 'bar'); -}; -like($@, qr/1 was expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/15_mounts.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/15_mounts.t (revision 24529) +++ (revision ) @@ -1,67 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2008 Joshua Hoblitt -# -# $Id: 15_mounts.t,v 1.3 2008-09-11 22:35:52 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 18; - -use lib qw( ./t ./lib ); - -use Nebulous::Server; -use Test::URI; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -Test::Nebulous->setup; - -{ - # key - my $mounts = $neb->mounts(); - - is(scalar @$mounts, 7, "number of rows"); - - my %row; - # first row - @row{qw(mountpoint total used vol_id name host path allocate available xattr)} - = @{$mounts->[0]}; - - is($row{total}, 100000000000); - is($row{used}, 100000000); - is($row{vol_id}, 1); - is($row{name}, "node01"); - is($row{host}, "node01"); - is($row{allocate}, 1); - is($row{available}, 1); - is($row{xattr}, 0); - - # 2nd row - @row{qw(mountpoint total used vol_id name host path allocate available xattr)} - = @{$mounts->[1]}; - - is($row{total}, 100000000000); - is($row{used}, 1000000000); - is($row{vol_id}, 2); - is($row{name}, "node02"); - is($row{host}, "node02"); - is($row{allocate}, 1); - is($row{available}, 1); - is($row{xattr}, 0); -} - -Test::Nebulous->setup; - -eval { - $neb->mounts("foo"); -}; -like($@, qr/0 were expected/, "no params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/15_server_xattr.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/15_server_xattr.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/15_server_xattr.t (revision 24557) @@ -0,0 +1,286 @@ +#!/usr/bin/perl + +# Copryight (C) 2007 Joshua Hoblitt +# +# $Id: 14_server_xattr.t,v 1.8 2008-07-09 23:32:35 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 44; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +# 1 key / xattr + +Test::Nebulous->setup; + +{ + my $uri = $neb->create_object('foo'); + + ok($neb->setxattr_object('foo', 'user.bar', 'baz', 'create'), 'set object xattr'); + { + my $xattrs = $neb->listxattr_object('foo'); + is(scalar @$xattrs, 1, 'number of xattrs'); + is(@$xattrs[0], 'user.bar', 'xattr name'); + } + + my $value = $neb->getxattr_object('foo', 'user.bar'); + is($value, 'baz', 'xattr value'); + + ok($neb->removexattr_object('foo', 'user.bar'), "remove object xattr"); + { + my $xattrs = $neb->listxattr_object('foo'); + is(scalar @$xattrs, 0, 'number of xattrs'); + } +} + +# multiple xattrs + +Test::Nebulous->setup; + +{ + my $uri = $neb->create_object('foo'); + + ok($neb->setxattr_object('foo', 'user.bar', 'baz', 'create'), 'set object xattr'); + ok($neb->setxattr_object('foo', 'user.bonk', 'quix', 'create'), 'set object xattr'); + + { + my $xattrs = $neb->listxattr_object('foo'); + is(scalar @$xattrs, 2, 'number of xattrs'); + is(@$xattrs[0], 'user.bar', 'xattr name'); + is(@$xattrs[1], 'user.bonk', 'xattr name'); + } + + my $value = $neb->getxattr_object('foo', 'user.bar'); + is($value, 'baz', 'xattr value'); + $value = $neb->getxattr_object('foo', 'user.bonk'); + is($value, 'quix', 'xattr value'); + + ok($neb->removexattr_object('foo', 'user.bar'), "remove object xattr"); + ok($neb->removexattr_object('foo', 'user.bonk'), "remove object xattr"); + { + my $xattrs = $neb->listxattr_object('foo'); + is(scalar @$xattrs, 0, 'number of xattrs'); + } +} + +# replace xattrs + +Test::Nebulous->setup; + +{ + my $uri = $neb->create_object('foo'); + + ok($neb->setxattr_object('foo', 'user.bar', 'baz', 'create'), 'set object xattr'); + ok($neb->setxattr_object('foo', 'user.bar', 'quix', 'replace'), 're-set object xattr'); + + { + my $xattrs = $neb->listxattr_object('foo'); + is(scalar @$xattrs, 1, 'number of xattrs'); + is(@$xattrs[0], 'user.bar', 'xattr name'); + } + + my $value = $neb->getxattr_object('foo', 'user.bar'); + is($value, 'quix', 'xattr value'); + + ok($neb->removexattr_object('foo', 'user.bar'), "remove object xattr"); + { + my $xattrs = $neb->listxattr_object('foo'); + is(scalar @$xattrs, 0, 'number of xattrs'); + } +} + +# setxattr_object + +Test::Nebulous->setup; + +eval { + $neb->setxattr_object('foo', 'user.bar', 'baz', 'create'); +}; +like($@, qr/is valid object key/, "create xattr on non-existant key"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->setxattr_object('foo', 'luser.bar', 'baz', 'create'); +}; +like($@, qr/xattr is in user. namespace/, "user. namspace"); + +Test::Nebulous->setup; + +eval { + $neb->setxattr_object('foo', 'user.bar', 'baz', 'replace'); +}; +like($@, qr/is valid object key/, "replace xattr on non-existant key"); + +Test::Nebulous->setup; + +eval { + $neb->setxattr_object(); +}; +like($@, qr/4 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->setxattr_object('foo', 'user.bar'); +}; +like($@, qr/4 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->setxattr_object('foo', 'user.bar', 'baz'); +}; +like($@, qr/4 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->setxattr_object('foo', 'user.bar', 'baz', 'create', 'quix'); +}; +like($@, qr/4 were expected/, "too many params"); + +# getxattr_object + +Test::Nebulous->setup; + +eval { + $neb->getxattr_object('foo', 'bar'); +}; +like($@, qr/is valid object key/, "get xattr from non-existant nebulous key"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->getxattr_object('foo', 'luser.bar'); +}; +like($@, qr/xattr is in user. namespace/, "user. namespace"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + $neb->getxattr_object('foo', 'user.bar'); +}; +like($@, qr|xattr neb:///foo:user.bar does not exist|, + "get xattr from non-existant xattr key"); + +Test::Nebulous->setup; + +eval { + $neb->getxattr_object(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->getxattr_object('foo'); +}; +like($@, qr/2 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->getxattr_object('foo', 'user.bar', 'baz'); +}; +like($@, qr/2 were expected/, "too many params"); + +# listxattr_object + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->listxattr_object(); +}; +like($@, qr/1 was expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->listxattr_object('foo', 'bar'); +}; +like($@, qr/1 was expected/, "too many params"); + +# removexattr_object + +Test::Nebulous->setup; + +eval { + $neb->removexattr_object('foo', 'user.bar'); +}; +like($@, qr/is valid object key/, "remove xattr from non-existant nebulous key"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->getxattr_object('foo', 'luser.bar'); +}; +like($@, qr/xattr is in user. namespace/, "user. namespace"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + $neb->removexattr_object('foo', 'user.bar'); +}; +like($@, qr|xattr neb:///foo:user.bar does not exist|, + "remove xattr from non-existant xattr key"); + +Test::Nebulous->setup; + +eval { + $neb->removexattr_object(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->removexattr_object('foo'); +}; +like($@, qr/2 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->removexattr_object('foo', 'user.bar', 'baz'); +}; +like($@, qr/2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/16_mounts.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/16_mounts.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/16_mounts.t (revision 24557) @@ -0,0 +1,67 @@ +#!/usr/bin/perl + +# Copryight (C) 2008 Joshua Hoblitt +# +# $Id: 15_mounts.t,v 1.3 2008-09-11 22:35:52 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 18; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::URI; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +{ + # key + my $mounts = $neb->mounts(); + + is(scalar @$mounts, 7, "number of rows"); + + my %row; + # first row + @row{qw(mountpoint total used vol_id name host path allocate available xattr)} + = @{$mounts->[0]}; + + is($row{total}, 100000000000); + is($row{used}, 100000000); + is($row{vol_id}, 1); + is($row{name}, "node01"); + is($row{host}, "node01"); + is($row{allocate}, 1); + is($row{available}, 1); + is($row{xattr}, 0); + + # 2nd row + @row{qw(mountpoint total used vol_id name host path allocate available xattr)} + = @{$mounts->[1]}; + + is($row{total}, 100000000000); + is($row{used}, 1000000000); + is($row{vol_id}, 2); + is($row{name}, "node02"); + is($row{host}, "node02"); + is($row{allocate}, 1); + is($row{available}, 1); + is($row{xattr}, 0); +} + +Test::Nebulous->setup; + +eval { + $neb->mounts("foo"); +}; +like($@, qr/0 were expected/, "no params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/16_server_swap_objects.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/16_server_swap_objects.t (revision 24529) +++ (revision ) @@ -1,117 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2007 Joshua Hoblitt -# -# $Id: 16_server_swap_objects.t,v 1.1 2008-10-13 20:41:17 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Test::More tests => 13; - -use lib qw( ./t ./lib ); - -use File::Basename qw( basename ); -use Nebulous::Server; -use Test::Nebulous; - -my $neb = Nebulous::Server->new( - dsn => $NEB_DB, - dbuser => $NEB_USER, - dbpasswd => $NEB_PASS, -); - -use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; - -Test::Nebulous->setup; - -{ - my $key1 = "foo1"; - my $key2 = "foo2"; - my $uri1 = $neb->create_object($key1); - my $uri2 = $neb->create_object($key2); - - ok($neb->swap_objects($key1, $key2), "swap succeeded"); - - my $new_uri1 = ($neb->find_instances($key1))->[0]; - my $new_uri2 = ($neb->find_instances($key2))->[0]; - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1], - storage_object => [so_id => 2, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 1], - ); - - is($uri1, $new_uri2, "key1 -> key2"); - is($uri2, $new_uri1, "key2 -> key1"); -} - -Test::Nebulous->setup; - -{ - my $key1 = "foo1"; - my $key2 = "a/foo2"; - my $uri1 = $neb->create_object($key1); - my $uri2 = $neb->create_object($key2); - - ok($neb->swap_objects($key1, $key2), "swap succeeded"); - - my $new_uri1 = ($neb->find_instances($key1))->[0]; - my $new_uri2 = ($neb->find_instances($key2))->[0]; - - expected_dataset_ok( - directory => [dir_id => 1, dirname => '/', parent_id => 1], - directory => [dir_id => 2, dirname => 'a', parent_id => 1], - storage_object => [so_id => 1, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2], - storage_object => [so_id => 2, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 1], - ); - - is($uri1, $new_uri2, "key1 -> key2"); - is($uri2, $new_uri1, "key2 -> key1"); -} - -Test::Nebulous->setup; - -eval { - $neb->create_object('bar'); - - $neb->swap_objects('foo', 'bar'); -}; -like($@, qr/is valid object key/, "key1 doesn't exist"); - -Test::Nebulous->setup; - -eval { - $neb->create_object('foo'); - - $neb->swap_objects('foo', 'bar'); -}; -like($@, qr/is valid object key/, "key2 doesn't exist"); - -Test::Nebulous->setup; - -eval { - $neb->swap_objects(); -}; -like($@, qr/2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - - $neb->swap_objects("foo"); -}; -like($@, qr/2 were expected/, "too few params"); - -Test::Nebulous->setup; - -eval { - $neb->create_object("foo"); - $neb->create_object("bar"); - - $neb->swap_objects("foo", "bar", "baz"); -}; -like($@, qr/2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/17_server_swap_objects.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/17_server_swap_objects.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/17_server_swap_objects.t (revision 24557) @@ -0,0 +1,117 @@ +#!/usr/bin/perl + +# Copryight (C) 2007 Joshua Hoblitt +# +# $Id: 16_server_swap_objects.t,v 1.1 2008-10-13 20:41:17 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 13; + +use lib qw( ./t ./lib ); + +use File::Basename qw( basename ); +use Nebulous::Server; +use Test::Nebulous; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS; + +Test::Nebulous->setup; + +{ + my $key1 = "foo1"; + my $key2 = "foo2"; + my $uri1 = $neb->create_object($key1); + my $uri2 = $neb->create_object($key2); + + ok($neb->swap_objects($key1, $key2), "swap succeeded"); + + my $new_uri1 = ($neb->find_instances($key1))->[0]; + my $new_uri2 = ($neb->find_instances($key2))->[0]; + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1], + storage_object => [so_id => 2, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 1], + ); + + is($uri1, $new_uri2, "key1 -> key2"); + is($uri2, $new_uri1, "key2 -> key1"); +} + +Test::Nebulous->setup; + +{ + my $key1 = "foo1"; + my $key2 = "a/foo2"; + my $uri1 = $neb->create_object($key1); + my $uri2 = $neb->create_object($key2); + + ok($neb->swap_objects($key1, $key2), "swap succeeded"); + + my $new_uri1 = ($neb->find_instances($key1))->[0]; + my $new_uri2 = ($neb->find_instances($key2))->[0]; + + expected_dataset_ok( + directory => [dir_id => 1, dirname => '/', parent_id => 1], + directory => [dir_id => 2, dirname => 'a', parent_id => 1], + storage_object => [so_id => 1, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2], + storage_object => [so_id => 2, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 1], + ); + + is($uri1, $new_uri2, "key1 -> key2"); + is($uri2, $new_uri1, "key2 -> key1"); +} + +Test::Nebulous->setup; + +eval { + $neb->create_object('bar'); + + $neb->swap_objects('foo', 'bar'); +}; +like($@, qr/is valid object key/, "key1 doesn't exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object('foo'); + + $neb->swap_objects('foo', 'bar'); +}; +like($@, qr/is valid object key/, "key2 doesn't exist"); + +Test::Nebulous->setup; + +eval { + $neb->swap_objects(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->swap_objects("foo"); +}; +like($@, qr/2 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + $neb->create_object("bar"); + + $neb->swap_objects("foo", "bar", "baz"); +}; +like($@, qr/2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/18_server_chmod_object.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/18_server_chmod_object.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous-Server/t/18_server_chmod_object.t (revision 24557) @@ -0,0 +1,111 @@ +#!/usr/bin/perl + +# Copryight (C) 2009 Joshua Hoblitt + +use strict; +use warnings FATAL => qw( all ); + +use Test::More tests => 13; + +use lib qw( ./t ./lib ); + +use Nebulous::Server; +use Test::URI; +use Test::Nebulous; +use URI; +use File::stat; + +my $neb = Nebulous::Server->new( + dsn => $NEB_DB, + dbuser => $NEB_USER, + dbpasswd => $NEB_PASS, +); + +Test::Nebulous->setup; + +# object with one instance +{ + my $key = "foo"; + my $uri = $neb->create_object($key); + + my $path = URI->new($uri)->path; + my $st1 = stat($path); + + my $mode = $neb->chmod_object($key, 0440); + my $st2 = stat($path); + + is($st2->mode &07777, 0440, "chmod() single instance"); + is($mode, 0440, "returned mode"); + is($neb->getxattr_object($key, 'user.mode'), 0440, "xattr user.mode"); +} + +Test::Nebulous->setup; + +# object with two instances +{ + my $key = "foo"; + my $uri1 = $neb->create_object($key); + my $uri2 = $neb->replicate_object($key); + + my $mode = $neb->chmod_object($key, 0440); + + my $st1 = stat(URI->new($uri1)->path); + my $st2 = stat(URI->new($uri2)->path); + + is($st1->mode &07777, 0440, "chmod() first instance"); + is($st2->mode &07777, 0440, "chmod() second instance"); + is($mode, 0440, "returned mode"); + is($neb->getxattr_object($key, 'user.mode'), 0440, "xattr user.mode"); +} + +Test::Nebulous->setup; + +eval { + $neb->chmod_object("foo", 0644); +}; +like($@, qr/valid object/, "object does not exist"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->chmod_object("foo", 0640); +}; +like($@, qr/allowable mode/, "mode is not 0400"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->chmod_object("foo", 99999); +}; +like($@, qr/allowable mode/, "bad mode"); + +Test::Nebulous->setup; + +eval { + $neb->chmod_object(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->chmod_object("foo"); +}; +like($@, qr/2 were expected/, "one param"); + +Test::Nebulous->setup; + +eval { + $neb->create_object("foo"); + + $neb->chmod_object("foo", 0440, 2); +}; +like($@, qr/2 were expected/, "three params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous-Server/t/75_parse_neb_key.t =================================================================== --- branches/eam_branches/20090522/Nebulous-Server/t/75_parse_neb_key.t (revision 24529) +++ branches/eam_branches/20090522/Nebulous-Server/t/75_parse_neb_key.t (revision 24557) @@ -10,5 +10,5 @@ use Test::More; -plan tests => 99; +plan tests => 106; use lib qw( ./t ./lib ); @@ -28,5 +28,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -36,5 +36,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -44,5 +44,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -52,5 +52,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -60,5 +60,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -68,5 +68,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -77,5 +77,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, 'boing', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -85,5 +85,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, 'boing', 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, 1, 'soft volume name'); } @@ -94,5 +94,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -102,5 +102,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -110,5 +110,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -118,5 +118,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -126,5 +126,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -135,5 +135,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, 1, 'soft volume name'); } @@ -144,5 +144,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'boing', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -152,5 +152,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'boing', 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, 1, 'soft volume name'); } @@ -160,5 +160,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, 1, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -169,5 +169,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -178,5 +178,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -186,5 +186,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -195,5 +195,5 @@ is($key->path, 'bar/baz/quix', 'path'); is($key->volume, 'foo', 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, 1, 'soft volume name'); } @@ -205,5 +205,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -213,5 +213,5 @@ is($key->volume, undef, 'volume name'); is($key->path, 'foo/bar/baz/quix', 'path'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -221,5 +221,5 @@ is($key->path, 'foo/bar/baz/quix', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -230,5 +230,5 @@ is($key->path, '', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -238,5 +238,5 @@ is($key->path, '', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -246,5 +246,5 @@ is($key->path, '', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -254,5 +254,5 @@ is($key->path, '', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -262,5 +262,5 @@ is($key->path, '', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); } @@ -270,5 +270,28 @@ is($key->path, '', 'path'); is($key->volume, undef, 'volume name'); - is($key->soft_volume, undef, 'soft volume name'); + is($key->hard_volume, undef, 'soft volume name'); +} + +# stringification + +{ + my $txt = 'neb:///bar/baz'; + my $key = parse_neb_key($txt); + + is("$key", $txt, "stringified"); +} + +{ + my $txt = 'neb://foo.0/bar/baz'; + my $key = parse_neb_key($txt); + + is("$key", $txt, "stringified"); +} + +{ + my $txt = 'neb://~foo.0/bar/baz'; + my $key = parse_neb_key($txt); + + is("$key", $txt, "stringified"); } @@ -322,2 +345,23 @@ }; like( $@, qr/requires a path/, "no path" ); + +# params +eval { + my $key = parse_neb_key(); +}; +like( $@, qr/key param is not optional/, "no params" ); + +eval { + my $key = parse_neb_key(undef); +}; +like( $@, qr/key param is not optional/, "key is undef" ); + +eval { + my $key = parse_neb_key(undef, 'bar'); +}; +like( $@, qr/key param is not optional/, "key is undef" ); + +eval { + my $key = parse_neb_key('foo', 'bar', 'foo'); +}; +like( $@, qr/too many params/, "too many params" ); Index: branches/eam_branches/20090522/Nebulous/Changes =================================================================== --- branches/eam_branches/20090522/Nebulous/Changes (revision 24529) +++ branches/eam_branches/20090522/Nebulous/Changes (revision 24557) @@ -1,3 +1,9 @@ Revision history for Nebulous + +0.17 + - add chmod() API + - change neb-ls to always output in --long format + - add neb-rm --force flag + - change neb-locate's output seperator to \n 0.10 @@ -9,4 +15,5 @@ - add neb-swap - add nebclient:nebSetXattr() + - add neb-rm --move flag 0.09 Wed Jul 9 16:36:27 HST 2008 Index: branches/eam_branches/20090522/Nebulous/MANIFEST =================================================================== --- branches/eam_branches/20090522/Nebulous/MANIFEST (revision 24529) +++ branches/eam_branches/20090522/Nebulous/MANIFEST (revision 24557) @@ -79,6 +79,6 @@ t/00_distribution.t t/01_load.t -t/50_client_new.t -t/51_client_create.t +t/49_client_new.t +t/50_client_create.t t/51_client_open_create.t t/52_client_replicate.t @@ -98,4 +98,5 @@ t/66_client_xattr.t t/67_client_swap.t +t/68_client_chmod.t t/70_neb-ls.t t/90_nebclient.t Index: branches/eam_branches/20090522/Nebulous/bin/neb-locate =================================================================== --- branches/eam_branches/20090522/Nebulous/bin/neb-locate (revision 24529) +++ branches/eam_branches/20090522/Nebulous/bin/neb-locate (revision 24557) @@ -41,4 +41,6 @@ unless defined $neb; +$volume = 'any' if defined $all; + my $uris; if (defined $volume) { @@ -70,5 +72,5 @@ } -print join(" ", @files), "\n"; +print join("\n", @files), "\n"; __END__ Index: branches/eam_branches/20090522/Nebulous/bin/neb-ls =================================================================== --- branches/eam_branches/20090522/Nebulous/bin/neb-ls (revision 24529) +++ branches/eam_branches/20090522/Nebulous/bin/neb-ls (revision 24557) @@ -21,4 +21,7 @@ $server = $ENV{'NEB_SERVER'} unless $server; + +# make --long the default +$long = 1; GetOptions( Index: branches/eam_branches/20090522/Nebulous/bin/neb-replicate =================================================================== --- branches/eam_branches/20090522/Nebulous/bin/neb-replicate (revision 24529) +++ branches/eam_branches/20090522/Nebulous/bin/neb-replicate (revision 24557) @@ -16,5 +16,5 @@ use Pod::Usage qw( pod2usage ); -my ($server, $volume, $copies); +my ($server, $volume, $copies, $set_copies); $copies = 1; # default to making just one copy Index: branches/eam_branches/20090522/Nebulous/bin/neb-rm =================================================================== --- branches/eam_branches/20090522/Nebulous/bin/neb-rm (revision 24529) +++ branches/eam_branches/20090522/Nebulous/bin/neb-rm (revision 24557) @@ -16,9 +16,15 @@ use Pod::Usage qw( pod2usage ); -my ($server); +my ( + $force, + $move, + $server, +); $server = $ENV{'NEB_SERVER'} unless $server; GetOptions( + 'force|f' => \$force, + 'move|m' => \$move, 'server|s=s' => \$server, ) || pod2usage( 2 ); @@ -35,5 +41,9 @@ foreach my $key (@ARGV) { - $neb->delete($key) or warn "failed to delete key: $key"; + unless (defined $move) { + $neb->delete($key) or warn "failed to delete key: $key"; + } else { + $neb->move($key, "${key}.rm") or warn "failed to move key: $key"; + } } @@ -48,5 +58,5 @@ =head1 SYNOPSIS - neb-rm [--server ] + neb-rm [--server ] [--force] [--move] =head1 DESCRIPTION @@ -58,4 +68,18 @@ =over 4 + +=item * --force|-f + +When set, C<> will be removed from the nebulous database even if the on +disk file(s) are missing. + +Optional. + +=item * --move|-m + +When set, this flag causes C<> to be renamed to C<.rm> instead of +being deleted. + +Optional. =item * --server|-s Index: branches/eam_branches/20090522/Nebulous/bin/neb-stat =================================================================== --- branches/eam_branches/20090522/Nebulous/bin/neb-stat (revision 24529) +++ branches/eam_branches/20090522/Nebulous/bin/neb-stat (revision 24557) @@ -41,5 +41,5 @@ my $stat = $neb->stat($key); die "nebulous key: $key not found" unless $stat; -my $instances = $neb->find_instances($key, ':any'); +my $instances = $neb->find_instances($key, 'any'); die "no instances found" unless $instances; Index: branches/eam_branches/20090522/Nebulous/lib/Nebulous/Client.pm =================================================================== --- branches/eam_branches/20090522/Nebulous/lib/Nebulous/Client.pm (revision 24529) +++ branches/eam_branches/20090522/Nebulous/lib/Nebulous/Client.pm (revision 24557) @@ -9,5 +9,5 @@ no warnings qw( uninitialized ); -our $VERSION = '0.09'; +our $VERSION = '0.17'; use Digest::MD5; @@ -16,5 +16,5 @@ use Nebulous::Client::Log; use Nebulous::Util qw( :standard ); -use Params::Validate qw( validate validate_pos SCALAR UNDEF ); +use Params::Validate qw( validate validate_pos SCALAR UNDEF BOOLEAN ); #use SOAP::Lite +trace => [qw( debug )]; use SOAP::Lite; @@ -329,4 +329,11 @@ my $uri = $self->delete_instance($key, @$locations[0]); + eval { + _nuke_file(_get_file_path(@$locations[0])); + }; + if ($@) { + $log->logdie($@); + } + $log->debug("leaving"); @@ -803,7 +810,17 @@ my $self = shift; - my ( $key ) = validate_pos( @_, - { - type => SCALAR, + my ($key, $force) = validate_pos( @_, + { + type => SCALAR, + }, + { + type => BOOLEAN, + optional => 1, + default => undef, + callbacks => { + 'is boolean' => sub { + $_[0] == 0 or $_[0] == 1 or $_[0] == undef; + }, + }, }, ); @@ -817,5 +834,32 @@ # a lock is implicitly removed when the last storage object is deleted foreach my $uri ( @$locations ) { + # it is being assumed here that it is better to have files on disk and + # not in the database then the inverse. + my $path; + eval { + $path = _get_file_path( $uri ); + }; + if ($@) { + if ($force) { + $log->warn($@); + $log->warn("exception ignored because force is in effect"); + } else { + $log->logdie($@); + } + } + $self->delete_instance($key, $uri) or return undef; + + eval { + _nuke_file( $path ); + }; + if ($@) { + if ($force) { + $log->warn($@); + $log->warn("exception ignored because force is in effect"); + } else { + $log->logdie($@); + } + } } @@ -959,13 +1003,4 @@ $log->debug( "entered - @_" ); - # it is being assumed here that it is better to have files on disk and not in - # the database then the inverse. - - my $path; - eval { - $path = _get_file_path( $uri ); - }; - $log->logdie( $@ ) if $@; - my $response = $self->{ 'server' }->delete_instance($key, $uri); if ( $response->fault ) { @@ -982,9 +1017,4 @@ $log->debug( "server deleted instance" ); - eval { - _nuke_file( $path ); - }; - $log->logdie( $@ ) if $@; - $log->debug( "leaving" ); @@ -1045,4 +1075,43 @@ return $stats; +} + + +sub chmod +{ + my $self = shift; + + my ($key, $mode) = validate_pos( @_, + { + type => SCALAR, + }, + { + type => SCALAR, + regex => qr/\d{3,4}/, + }, + ); + + $log->debug( "entered - @_" ); + + my $response = $self->{ 'server' }->chmod_object($key, $mode); + if ( $response->fault ) { + $self->set_err($response->faultstring); + if ($response->faultstring =~ /is valid object key/) { + $log->debug( "leaving" ); + return; + } + if ($response->faultstring =~ /is allowable mode/) { + $log->debug( "leaving" ); + return; + } + + $log->logdie("unhandled fault - ", $self->err); + } + + my $response_mode = $response->result; + + $log->debug( "leaving" ); + + return $response_mode; } Index: branches/eam_branches/20090522/Nebulous/lib/Nebulous/Client.pod =================================================================== --- branches/eam_branches/20090522/Nebulous/lib/Nebulous/Client.pod (revision 24529) +++ branches/eam_branches/20090522/Nebulous/lib/Nebulous/Client.pod (revision 24557) @@ -13,10 +13,10 @@ ); - my $uri = $neb->create( "key", "node01" ); - my $fh = $neb->open_create( "key", "node01" ); - $neb->replicate( "key", "node01" ); - $neb->cull( "key", "node01" ); - $neb->lock( "key", 'write' ); - $neb->unlock( "key", 'write' ); + my $uri = $neb->create( $key, $vol ); + my $fh = $neb->open_create( $key, $vol ); + $neb->replicate( $key, $vol ); + $neb->cull( $key, $vol ); + $neb->lock( $key, 'write' ); + $neb->unlock( $key, 'write' ); $neb->setxattr( $key, $name, $value, $flags ); $neb->getxattr( $key, $name ); @@ -24,14 +24,15 @@ $neb->removexattr( $key, $name ); my $uris = $neb->find_objects( $pattern ); - my $uris = $neb->find_instances( "key", "node01" ); - my $path = $neb->find( "key" ); - my $fh = $neb->open( "key", 'read' ); - $neb->delete( "key" ); - $neb->copy( "key", "new_key", "node01" ); - $neb->move( "key", "new_key" ); - $neb->swap( "key1", "key2" ); + my $uris = $neb->find_instances( $key, $vol ); + my $path = $neb->find( $key ); + my $fh = $neb->open( $key, 'read' ); + $neb->delete( $key ); + $neb->copy( $key, $new_key, $vol ); + $neb->move( $key, $new_key ); + $neb->swap( $key1, $key2 ); $neb->delete_instance( $uri ); - my $stats = $neb->stat( "key" ); + my $stats = $neb->stat( $key ); my $mounts = $neb->mounts(); + my $mode = $neb->chmod( $key, $mode ); =head1 DESCRIPTION @@ -425,4 +426,11 @@ ] +=item * chmod( $key, $mode ); + +Accepts 2 parameters, both mandatory. C<$key> is the nebulous key to operate +on and C<$mode> are the POSIX like file permissions to set on all instances of +C<$key>. C<$mode> must be specified in oct. Returns C<$mode> or sucess or an +exception on failure. + =back Index: branches/eam_branches/20090522/Nebulous/nebclient/nebulous.wsdl =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/nebulous.wsdl (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/nebulous.wsdl (revision 24557) @@ -140,4 +140,13 @@ + + + + + + + + + @@ -240,4 +249,11 @@ --> + + + + + @@ -390,4 +406,15 @@ + + + + + + + + + soap, server->endpoint, + NULL, (char *)key, mode, &response) != SOAP_OK) { + nebSetServerErr(server); + return -1; + } + + return 0; +} + + char *nebErr(nebServer *server) { Index: branches/eam_branches/20090522/Nebulous/nebclient/src/nebclient.h =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/nebclient.h (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/nebclient.h (revision 24557) @@ -11,4 +11,6 @@ #ifndef NEBCLIENT_H #define NEBCLIENT_H 1 + +// #include #ifdef __cplusplus @@ -296,4 +298,18 @@ ); +/** Change permissions of a storage objects + * + * This function will change the file permissions of all currently on disk + * instances of a storage object. + * + * @return the new mode or NULL on failure. + */ + +int nebChmod( + nebServer *server, ///< nebServer object + const char *key, /// storage object key (name) + mode_t mode /// chmod(2) compatible mode (mode_t) +); + /** Returns the error message from the last nebclient function that failed. * Index: branches/eam_branches/20090522/Nebulous/nebclient/src/nebulous.h =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/nebulous.h (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/nebulous.h (revision 24557) @@ -1,5 +1,5 @@ /* src/nebulous.h Generated by wsdl2h 1.2.12 from nebulous.wsdl and typemap.dat - 2009-04-20 22:15:21 GMT + 2009-06-01 23:55:50 GMT Copyright (C) 2001-2008 Robert van Engelen, Genivia Inc. All Rights Reserved. This part of the software is released under one of the following licenses: @@ -119,4 +119,5 @@ - @ref ns1__delete_USCOREinstance - @ref ns1__stat_USCOREobject + - @ref ns1__chmod_USCOREobject @section SOAP_ports Endpoints of Binding "SOAP" @@ -893,3 +894,56 @@ ); +/******************************************************************************\ + * * + * ns1__chmod_USCOREobject * + * * +\******************************************************************************/ + + +/// Operation "ns1__chmod_USCOREobject" of service binding "SOAP" + +/** + +Operation details: + + - SOAP RPC encodingStyle="http://schemas.xmlsoap.org/soap/encoding/" + - SOAP action="urn:Nebulous/Server/SOAP#chmod_object" + +C stub function (defined in soapClient.c[pp] generated by soapcpp2): +@code + int soap_call_ns1__chmod_USCOREobject( + struct soap *soap, + NULL, // char *endpoint = NULL selects default endpoint for this operation + NULL, // char *action = NULL selects default action for this operation + // request parameters: + char* key, + int mode, + // response parameters: + int *result + ); +@endcode + +C server function (called from the service dispatcher defined in soapServer.c[pp]): +@code + int ns1__chmod_USCOREobject( + struct soap *soap, + // request parameters: + char* key, + int mode, + // response parameters: + int *result + ); +@endcode + +*/ + +//gsoap ns1 service method-style: chmod_USCOREobject rpc +//gsoap ns1 service method-encoding: chmod_USCOREobject http://schemas.xmlsoap.org/soap/encoding/ +//gsoap ns1 service method-action: chmod_USCOREobject urn:Nebulous/Server/SOAP#chmod_object +int ns1__chmod_USCOREobject( + char* key, ///< Request parameter + int mode, ///< Request parameter + int *result ///< Response parameter +); + /* End of src/nebulous.h */ Index: branches/eam_branches/20090522/Nebulous/nebclient/src/soapC.c =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/soapC.c (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/soapC.c (revision 24557) @@ -12,5 +12,5 @@ #endif -SOAP_SOURCE_STAMP("@(#) soapC.c ver 2.7.12 2009-04-20 22:15:22 GMT") +SOAP_SOURCE_STAMP("@(#) soapC.c ver 2.7.12 2009-06-01 23:55:50 GMT") @@ -161,4 +161,8 @@ case SOAP_TYPE_int: return soap_in_int(soap, NULL, NULL, "xsd:int"); + case SOAP_TYPE_ns1__chmod_USCOREobject: + return soap_in_ns1__chmod_USCOREobject(soap, NULL, NULL, "ns1:chmod_object"); + case SOAP_TYPE_ns1__chmod_USCOREobjectResponse: + return soap_in_ns1__chmod_USCOREobjectResponse(soap, NULL, NULL, "ns1:chmod_objectResponse"); case SOAP_TYPE_ns1__stat_USCOREobject: return soap_in_ns1__stat_USCOREobject(soap, NULL, NULL, "ns1:stat_object"); @@ -250,4 +254,12 @@ return soap_in_int(soap, NULL, NULL, NULL); } + if (!soap_match_tag(soap, t, "ns1:chmod_object")) + { *type = SOAP_TYPE_ns1__chmod_USCOREobject; + return soap_in_ns1__chmod_USCOREobject(soap, NULL, NULL, NULL); + } + if (!soap_match_tag(soap, t, "ns1:chmod_objectResponse")) + { *type = SOAP_TYPE_ns1__chmod_USCOREobjectResponse; + return soap_in_ns1__chmod_USCOREobjectResponse(soap, NULL, NULL, NULL); + } if (!soap_match_tag(soap, t, "ns1:stat_object")) { *type = SOAP_TYPE_ns1__stat_USCOREobject; @@ -440,4 +452,8 @@ case SOAP_TYPE_int: return soap_out_int(soap, tag, id, (const int *)ptr, "xsd:int"); + case SOAP_TYPE_ns1__chmod_USCOREobject: + return soap_out_ns1__chmod_USCOREobject(soap, tag, id, (const struct ns1__chmod_USCOREobject *)ptr, "ns1:chmod_object"); + case SOAP_TYPE_ns1__chmod_USCOREobjectResponse: + return soap_out_ns1__chmod_USCOREobjectResponse(soap, tag, id, (const struct ns1__chmod_USCOREobjectResponse *)ptr, "ns1:chmod_objectResponse"); case SOAP_TYPE_ns1__stat_USCOREobject: return soap_out_ns1__stat_USCOREobject(soap, tag, id, (const struct ns1__stat_USCOREobject *)ptr, "ns1:stat_object"); @@ -527,4 +543,10 @@ switch (type) { + case SOAP_TYPE_ns1__chmod_USCOREobject: + soap_serialize_ns1__chmod_USCOREobject(soap, (const struct ns1__chmod_USCOREobject *)ptr); + break; + case SOAP_TYPE_ns1__chmod_USCOREobjectResponse: + soap_serialize_ns1__chmod_USCOREobjectResponse(soap, (const struct ns1__chmod_USCOREobjectResponse *)ptr); + break; case SOAP_TYPE_ns1__stat_USCOREobject: soap_serialize_ns1__stat_USCOREobject(soap, (const struct ns1__stat_USCOREobject *)ptr); @@ -1195,4 +1217,164 @@ #endif +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__chmod_USCOREobject(struct soap *soap, struct ns1__chmod_USCOREobject *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + soap_default_string(soap, &a->key); + soap_default_int(soap, &a->mode); +} + +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__chmod_USCOREobject(struct soap *soap, const struct ns1__chmod_USCOREobject *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + soap_serialize_string(soap, &a->key); + soap_embedded(soap, &a->mode, SOAP_TYPE_int); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__chmod_USCOREobject(struct soap *soap, const struct ns1__chmod_USCOREobject *a, const char *tag, const char *type) +{ + register int id = soap_embed(soap, (void*)a, NULL, 0, tag, SOAP_TYPE_ns1__chmod_USCOREobject); + if (soap_out_ns1__chmod_USCOREobject(soap, tag, id, a, type)) + return soap->error; + return soap_putindependent(soap); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__chmod_USCOREobject(struct soap *soap, const char *tag, int id, const struct ns1__chmod_USCOREobject *a, const char *type) +{ + if (soap_element_begin_out(soap, tag, soap_embedded_id(soap, id, a, SOAP_TYPE_ns1__chmod_USCOREobject), type)) + return soap->error; + if (soap_out_string(soap, "key", -1, &a->key, "")) + return soap->error; + if (soap_out_int(soap, "mode", -1, &a->mode, "")) + return soap->error; + return soap_element_end_out(soap, tag); +} + +SOAP_FMAC3 struct ns1__chmod_USCOREobject * SOAP_FMAC4 soap_get_ns1__chmod_USCOREobject(struct soap *soap, struct ns1__chmod_USCOREobject *p, const char *tag, const char *type) +{ + if ((p = soap_in_ns1__chmod_USCOREobject(soap, tag, p, type))) + if (soap_getindependent(soap)) + return NULL; + return p; +} + +SOAP_FMAC3 struct ns1__chmod_USCOREobject * SOAP_FMAC4 soap_in_ns1__chmod_USCOREobject(struct soap *soap, const char *tag, struct ns1__chmod_USCOREobject *a, const char *type) +{ + size_t soap_flag_key = 1; + size_t soap_flag_mode = 1; + if (soap_element_begin_in(soap, tag, 0, type)) + return NULL; + a = (struct ns1__chmod_USCOREobject *)soap_id_enter(soap, soap->id, a, SOAP_TYPE_ns1__chmod_USCOREobject, sizeof(struct ns1__chmod_USCOREobject), 0, NULL, NULL, NULL); + if (!a) + return NULL; + soap_default_ns1__chmod_USCOREobject(soap, a); + if (soap->body && !*soap->href) + { + for (;;) + { soap->error = SOAP_TAG_MISMATCH; + if (soap_flag_key && (soap->error == SOAP_TAG_MISMATCH || soap->error == SOAP_NO_TAG)) + if (soap_in_string(soap, "key", &a->key, "xsd:string")) + { soap_flag_key--; + continue; + } + if (soap_flag_mode && soap->error == SOAP_TAG_MISMATCH) + if (soap_in_int(soap, "mode", &a->mode, "xsd:int")) + { soap_flag_mode--; + continue; + } + if (soap->error == SOAP_TAG_MISMATCH) + soap->error = soap_ignore_element(soap); + if (soap->error == SOAP_NO_TAG) + break; + if (soap->error) + return NULL; + } + if (soap_element_end_in(soap, tag)) + return NULL; + } + else + { a = (struct ns1__chmod_USCOREobject *)soap_id_forward(soap, soap->href, (void*)a, 0, SOAP_TYPE_ns1__chmod_USCOREobject, 0, sizeof(struct ns1__chmod_USCOREobject), 0, NULL); + if (soap->body && soap_element_end_in(soap, tag)) + return NULL; + } + if ((soap->mode & SOAP_XML_STRICT) && (soap_flag_mode > 0)) + { soap->error = SOAP_OCCURS; + return NULL; + } + return a; +} + +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__chmod_USCOREobjectResponse(struct soap *soap, struct ns1__chmod_USCOREobjectResponse *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + a->result = NULL; +} + +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__chmod_USCOREobjectResponse(struct soap *soap, const struct ns1__chmod_USCOREobjectResponse *a) +{ + (void)soap; (void)a; /* appease -Wall -Werror */ + soap_serialize_PointerToint(soap, &a->result); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__chmod_USCOREobjectResponse(struct soap *soap, const struct ns1__chmod_USCOREobjectResponse *a, const char *tag, const char *type) +{ + register int id = soap_embed(soap, (void*)a, NULL, 0, tag, SOAP_TYPE_ns1__chmod_USCOREobjectResponse); + if (soap_out_ns1__chmod_USCOREobjectResponse(soap, tag, id, a, type)) + return soap->error; + return soap_putindependent(soap); +} + +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__chmod_USCOREobjectResponse(struct soap *soap, const char *tag, int id, const struct ns1__chmod_USCOREobjectResponse *a, const char *type) +{ + if (soap_element_begin_out(soap, tag, soap_embedded_id(soap, id, a, SOAP_TYPE_ns1__chmod_USCOREobjectResponse), type)) + return soap->error; + if (soap_out_PointerToint(soap, "result", -1, &a->result, "")) + return soap->error; + return soap_element_end_out(soap, tag); +} + +SOAP_FMAC3 struct ns1__chmod_USCOREobjectResponse * SOAP_FMAC4 soap_get_ns1__chmod_USCOREobjectResponse(struct soap *soap, struct ns1__chmod_USCOREobjectResponse *p, const char *tag, const char *type) +{ + if ((p = soap_in_ns1__chmod_USCOREobjectResponse(soap, tag, p, type))) + if (soap_getindependent(soap)) + return NULL; + return p; +} + +SOAP_FMAC3 struct ns1__chmod_USCOREobjectResponse * SOAP_FMAC4 soap_in_ns1__chmod_USCOREobjectResponse(struct soap *soap, const char *tag, struct ns1__chmod_USCOREobjectResponse *a, const char *type) +{ + size_t soap_flag_result = 1; + if (soap_element_begin_in(soap, tag, 0, type)) + return NULL; + a = (struct ns1__chmod_USCOREobjectResponse *)soap_id_enter(soap, soap->id, a, SOAP_TYPE_ns1__chmod_USCOREobjectResponse, sizeof(struct ns1__chmod_USCOREobjectResponse), 0, NULL, NULL, NULL); + if (!a) + return NULL; + soap_default_ns1__chmod_USCOREobjectResponse(soap, a); + if (soap->body && !*soap->href) + { + for (;;) + { soap->error = SOAP_TAG_MISMATCH; + if (soap_flag_result && soap->error == SOAP_TAG_MISMATCH) + if (soap_in_PointerToint(soap, "result", &a->result, "xsd:int")) + { soap_flag_result--; + continue; + } + if (soap->error == SOAP_TAG_MISMATCH) + soap->error = soap_ignore_element(soap); + if (soap->error == SOAP_NO_TAG) + break; + if (soap->error) + return NULL; + } + if (soap_element_end_in(soap, tag)) + return NULL; + } + else + { a = (struct ns1__chmod_USCOREobjectResponse *)soap_id_forward(soap, soap->href, (void*)a, 0, SOAP_TYPE_ns1__chmod_USCOREobjectResponse, 0, sizeof(struct ns1__chmod_USCOREobjectResponse), 0, NULL); + if (soap->body && soap_element_end_in(soap, tag)) + return NULL; + } + return a; +} + SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__stat_USCOREobject(struct soap *soap, struct ns1__stat_USCOREobject *a) { Index: branches/eam_branches/20090522/Nebulous/nebclient/src/soapClient.c =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/soapClient.c (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/soapClient.c (revision 24557) @@ -10,5 +10,5 @@ #endif -SOAP_SOURCE_STAMP("@(#) soapClient.c ver 2.7.12 2009-04-20 22:15:21 GMT") +SOAP_SOURCE_STAMP("@(#) soapClient.c ver 2.7.12 2009-06-01 23:55:50 GMT") @@ -812,4 +812,62 @@ } +SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__chmod_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, int mode, int *result) +{ struct ns1__chmod_USCOREobject soap_tmp_ns1__chmod_USCOREobject; + struct ns1__chmod_USCOREobjectResponse *soap_tmp_ns1__chmod_USCOREobjectResponse; + if (!soap_endpoint) + soap_endpoint = "http://localhost:80/nebulous"; + if (!soap_action) + soap_action = "urn:Nebulous/Server/SOAP#chmod_object"; + soap->encodingStyle = "http://schemas.xmlsoap.org/soap/encoding/"; + soap_tmp_ns1__chmod_USCOREobject.key = key; + soap_tmp_ns1__chmod_USCOREobject.mode = mode; + soap_begin(soap); + soap_serializeheader(soap); + soap_serialize_ns1__chmod_USCOREobject(soap, &soap_tmp_ns1__chmod_USCOREobject); + if (soap_begin_count(soap)) + return soap->error; + if (soap->mode & SOAP_IO_LENGTH) + { if (soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__chmod_USCOREobject(soap, &soap_tmp_ns1__chmod_USCOREobject, "ns1:chmod_object", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap)) + return soap->error; + } + if (soap_end_count(soap)) + return soap->error; + if (soap_connect(soap, soap_endpoint, soap_action) + || soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__chmod_USCOREobject(soap, &soap_tmp_ns1__chmod_USCOREobject, "ns1:chmod_object", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap) + || soap_end_send(soap)) + return soap_closesock(soap); + if (!result) + return soap_closesock(soap); + soap_default_int(soap, result); + if (soap_begin_recv(soap) + || soap_envelope_begin_in(soap) + || soap_recv_header(soap) + || soap_body_begin_in(soap)) + return soap_closesock(soap); + soap_tmp_ns1__chmod_USCOREobjectResponse = soap_get_ns1__chmod_USCOREobjectResponse(soap, NULL, "ns1:chmod_objectResponse", ""); + if (soap->error) + { if (soap->error == SOAP_TAG_MISMATCH && soap->level == 2) + return soap_recv_fault(soap); + return soap_closesock(soap); + } + if (soap_body_end_in(soap) + || soap_envelope_end_in(soap) + || soap_end_recv(soap)) + return soap_closesock(soap); + if (result && soap_tmp_ns1__chmod_USCOREobjectResponse->result) + *result = *soap_tmp_ns1__chmod_USCOREobjectResponse->result; + return soap_closesock(soap); +} + #ifdef __cplusplus } Index: branches/eam_branches/20090522/Nebulous/nebclient/src/soapH.h =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/soapH.h (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/soapH.h (revision 24557) @@ -42,5 +42,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Fault -#define SOAP_TYPE_SOAP_ENV__Fault (64) +#define SOAP_TYPE_SOAP_ENV__Fault (67) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Fault(struct soap*, struct SOAP_ENV__Fault *); @@ -56,5 +56,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Reason -#define SOAP_TYPE_SOAP_ENV__Reason (63) +#define SOAP_TYPE_SOAP_ENV__Reason (66) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Reason(struct soap*, struct SOAP_ENV__Reason *); @@ -70,5 +70,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Detail -#define SOAP_TYPE_SOAP_ENV__Detail (60) +#define SOAP_TYPE_SOAP_ENV__Detail (63) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Detail(struct soap*, struct SOAP_ENV__Detail *); @@ -84,5 +84,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Code -#define SOAP_TYPE_SOAP_ENV__Code (58) +#define SOAP_TYPE_SOAP_ENV__Code (61) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Code(struct soap*, struct SOAP_ENV__Code *); @@ -98,5 +98,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Header -#define SOAP_TYPE_SOAP_ENV__Header (57) +#define SOAP_TYPE_SOAP_ENV__Header (60) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_default_SOAP_ENV__Header(struct soap*, struct SOAP_ENV__Header *); @@ -109,4 +109,24 @@ #endif +#ifndef SOAP_TYPE_ns1__chmod_USCOREobject +#define SOAP_TYPE_ns1__chmod_USCOREobject (59) +#endif +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__chmod_USCOREobject(struct soap*, struct ns1__chmod_USCOREobject *); +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__chmod_USCOREobject(struct soap*, const struct ns1__chmod_USCOREobject *); +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__chmod_USCOREobject(struct soap*, const struct ns1__chmod_USCOREobject *, const char*, const char*); +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__chmod_USCOREobject(struct soap*, const char*, int, const struct ns1__chmod_USCOREobject *, const char*); +SOAP_FMAC3 struct ns1__chmod_USCOREobject * SOAP_FMAC4 soap_get_ns1__chmod_USCOREobject(struct soap*, struct ns1__chmod_USCOREobject *, const char*, const char*); +SOAP_FMAC3 struct ns1__chmod_USCOREobject * SOAP_FMAC4 soap_in_ns1__chmod_USCOREobject(struct soap*, const char*, struct ns1__chmod_USCOREobject *, const char*); + +#ifndef SOAP_TYPE_ns1__chmod_USCOREobjectResponse +#define SOAP_TYPE_ns1__chmod_USCOREobjectResponse (58) +#endif +SOAP_FMAC3 void SOAP_FMAC4 soap_default_ns1__chmod_USCOREobjectResponse(struct soap*, struct ns1__chmod_USCOREobjectResponse *); +SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_ns1__chmod_USCOREobjectResponse(struct soap*, const struct ns1__chmod_USCOREobjectResponse *); +SOAP_FMAC3 int SOAP_FMAC4 soap_put_ns1__chmod_USCOREobjectResponse(struct soap*, const struct ns1__chmod_USCOREobjectResponse *, const char*, const char*); +SOAP_FMAC3 int SOAP_FMAC4 soap_out_ns1__chmod_USCOREobjectResponse(struct soap*, const char*, int, const struct ns1__chmod_USCOREobjectResponse *, const char*); +SOAP_FMAC3 struct ns1__chmod_USCOREobjectResponse * SOAP_FMAC4 soap_get_ns1__chmod_USCOREobjectResponse(struct soap*, struct ns1__chmod_USCOREobjectResponse *, const char*, const char*); +SOAP_FMAC3 struct ns1__chmod_USCOREobjectResponse * SOAP_FMAC4 soap_in_ns1__chmod_USCOREobjectResponse(struct soap*, const char*, struct ns1__chmod_USCOREobjectResponse *, const char*); + #ifndef SOAP_TYPE_ns1__stat_USCOREobject #define SOAP_TYPE_ns1__stat_USCOREobject (56) @@ -402,5 +422,5 @@ #ifndef SOAP_TYPE_PointerToSOAP_ENV__Reason -#define SOAP_TYPE_PointerToSOAP_ENV__Reason (66) +#define SOAP_TYPE_PointerToSOAP_ENV__Reason (69) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_PointerToSOAP_ENV__Reason(struct soap*, struct SOAP_ENV__Reason *const*); @@ -415,5 +435,5 @@ #ifndef SOAP_TYPE_PointerToSOAP_ENV__Detail -#define SOAP_TYPE_PointerToSOAP_ENV__Detail (65) +#define SOAP_TYPE_PointerToSOAP_ENV__Detail (68) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_PointerToSOAP_ENV__Detail(struct soap*, struct SOAP_ENV__Detail *const*); @@ -428,5 +448,5 @@ #ifndef SOAP_TYPE_PointerToSOAP_ENV__Code -#define SOAP_TYPE_PointerToSOAP_ENV__Code (59) +#define SOAP_TYPE_PointerToSOAP_ENV__Code (62) #endif SOAP_FMAC3 void SOAP_FMAC4 soap_serialize_PointerToSOAP_ENV__Code(struct soap*, struct SOAP_ENV__Code *const*); Index: branches/eam_branches/20090522/Nebulous/nebclient/src/soapServer.c =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/soapServer.c (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/soapServer.c (revision 24557) @@ -10,5 +10,5 @@ #endif -SOAP_SOURCE_STAMP("@(#) soapServer.c ver 2.7.12 2009-04-20 22:15:21 GMT") +SOAP_SOURCE_STAMP("@(#) soapServer.c ver 2.7.12 2009-06-01 23:55:50 GMT") @@ -105,4 +105,6 @@ if (!soap_match_tag(soap, soap->tag, "ns1:stat_object")) return soap_serve_ns1__stat_USCOREobject(soap); + if (!soap_match_tag(soap, soap->tag, "ns1:chmod_object")) + return soap_serve_ns1__chmod_USCOREobject(soap); return soap->error = SOAP_NO_METHOD; } @@ -713,4 +715,48 @@ } +SOAP_FMAC5 int SOAP_FMAC6 soap_serve_ns1__chmod_USCOREobject(struct soap *soap) +{ struct ns1__chmod_USCOREobject soap_tmp_ns1__chmod_USCOREobject; + struct ns1__chmod_USCOREobjectResponse soap_tmp_ns1__chmod_USCOREobjectResponse; + int soap_tmp_int; + soap_default_ns1__chmod_USCOREobjectResponse(soap, &soap_tmp_ns1__chmod_USCOREobjectResponse); + soap_default_int(soap, &soap_tmp_int); + soap_tmp_ns1__chmod_USCOREobjectResponse.result = &soap_tmp_int; + soap_default_ns1__chmod_USCOREobject(soap, &soap_tmp_ns1__chmod_USCOREobject); + soap->encodingStyle = "http://schemas.xmlsoap.org/soap/encoding/"; + if (!soap_get_ns1__chmod_USCOREobject(soap, &soap_tmp_ns1__chmod_USCOREobject, "ns1:chmod_object", NULL)) + return soap->error; + if (soap_body_end_in(soap) + || soap_envelope_end_in(soap) + || soap_end_recv(soap)) + return soap->error; + soap->error = ns1__chmod_USCOREobject(soap, soap_tmp_ns1__chmod_USCOREobject.key, soap_tmp_ns1__chmod_USCOREobject.mode, &soap_tmp_int); + if (soap->error) + return soap->error; + soap_serializeheader(soap); + soap_serialize_ns1__chmod_USCOREobjectResponse(soap, &soap_tmp_ns1__chmod_USCOREobjectResponse); + if (soap_begin_count(soap)) + return soap->error; + if (soap->mode & SOAP_IO_LENGTH) + { if (soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__chmod_USCOREobjectResponse(soap, &soap_tmp_ns1__chmod_USCOREobjectResponse, "ns1:chmod_objectResponse", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap)) + return soap->error; + }; + if (soap_end_count(soap) + || soap_response(soap, SOAP_OK) + || soap_envelope_begin_out(soap) + || soap_putheader(soap) + || soap_body_begin_out(soap) + || soap_put_ns1__chmod_USCOREobjectResponse(soap, &soap_tmp_ns1__chmod_USCOREobjectResponse, "ns1:chmod_objectResponse", "") + || soap_body_end_out(soap) + || soap_envelope_end_out(soap) + || soap_end_send(soap)) + return soap->error; + return soap_closesock(soap); +} + #ifdef __cplusplus } Index: branches/eam_branches/20090522/Nebulous/nebclient/src/soapStub.h =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/src/soapStub.h (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/src/soapStub.h (revision 24557) @@ -306,6 +306,25 @@ #endif +#ifndef SOAP_TYPE_ns1__chmod_USCOREobjectResponse +#define SOAP_TYPE_ns1__chmod_USCOREobjectResponse (58) +/* ns1:chmod_objectResponse */ +struct ns1__chmod_USCOREobjectResponse +{ + int *result; /* SOAP 1.2 RPC return element (when namespace qualified) */ /* optional element of type xsd:int */ +}; +#endif + +#ifndef SOAP_TYPE_ns1__chmod_USCOREobject +#define SOAP_TYPE_ns1__chmod_USCOREobject (59) +/* ns1:chmod_object */ +struct ns1__chmod_USCOREobject +{ + char *key; /* optional element of type xsd:string */ + int mode; /* required element of type xsd:int */ +}; +#endif + #ifndef SOAP_TYPE_SOAP_ENV__Header -#define SOAP_TYPE_SOAP_ENV__Header (57) +#define SOAP_TYPE_SOAP_ENV__Header (60) /* SOAP Header: */ struct SOAP_ENV__Header @@ -318,5 +337,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Code -#define SOAP_TYPE_SOAP_ENV__Code (58) +#define SOAP_TYPE_SOAP_ENV__Code (61) /* SOAP Fault Code: */ struct SOAP_ENV__Code @@ -328,5 +347,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Detail -#define SOAP_TYPE_SOAP_ENV__Detail (60) +#define SOAP_TYPE_SOAP_ENV__Detail (63) /* SOAP-ENV:Detail */ struct SOAP_ENV__Detail @@ -339,5 +358,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Reason -#define SOAP_TYPE_SOAP_ENV__Reason (63) +#define SOAP_TYPE_SOAP_ENV__Reason (66) /* SOAP-ENV:Reason */ struct SOAP_ENV__Reason @@ -348,5 +367,5 @@ #ifndef SOAP_TYPE_SOAP_ENV__Fault -#define SOAP_TYPE_SOAP_ENV__Fault (64) +#define SOAP_TYPE_SOAP_ENV__Fault (67) /* SOAP Fault: */ struct SOAP_ENV__Fault @@ -437,4 +456,6 @@ SOAP_FMAC5 int SOAP_FMAC6 ns1__stat_USCOREobject(struct soap*, char *key, struct ns1__stat_USCOREobjectResponse *_param_4); +SOAP_FMAC5 int SOAP_FMAC6 ns1__chmod_USCOREobject(struct soap*, char *key, int mode, int *result); + /******************************************************************************\ * * @@ -472,4 +493,6 @@ SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__stat_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, struct ns1__stat_USCOREobjectResponse *_param_4); +SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__chmod_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, int mode, int *result); + /******************************************************************************\ * * @@ -509,4 +532,6 @@ SOAP_FMAC5 int SOAP_FMAC6 soap_serve_ns1__stat_USCOREobject(struct soap*); + +SOAP_FMAC5 int SOAP_FMAC6 soap_serve_ns1__chmod_USCOREobject(struct soap*); #ifdef __cplusplus Index: branches/eam_branches/20090522/Nebulous/nebclient/tests/tests.c =================================================================== --- branches/eam_branches/20090522/Nebulous/nebclient/tests/tests.c (revision 24529) +++ branches/eam_branches/20090522/Nebulous/nebclient/tests/tests.c (revision 24557) @@ -20,5 +20,5 @@ char *key = "foobarbaz"; - plan_tests(28); + plan_tests(29); if (getenv("NEB_SERVER")) { @@ -136,4 +136,6 @@ neb_ok(server, nebMove(server, "copyiedfile", "movedfile"), "move object"); + neb_ok(server, nebChmod(server, "movedfile", 0440) == 0, "chmod object"); + if (!nebDelete(server, "movedfile")) { diag( "cleanup failed %s\n", nebErr(server)); Index: branches/eam_branches/20090522/Nebulous/t/49_client_new.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/49_client_new.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous/t/49_client_new.t (revision 24557) @@ -0,0 +1,60 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 50_client_new.t,v 1.1 2005-12-03 02:52:31 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Apache::Test qw( -withtestmore ); + +plan tests => 4; + +use lib qw( ./t ./lib ); + +use Nebulous::Client; +use Nebulous::Util qw( :standard ); +use Test::Nebulous; + +my $hostport = Apache::Test->config->{ 'hostport' }; + +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + isa_ok( $neb, "Nebulous::Client" ); +} + +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + uri => "http://example.com/PS/IPP/Nebulous/Client", + ); + + isa_ok( $neb, "Nebulous::Client" ); +} + +Test::Nebulous->setup; + +eval { + Nebulous::Client->new; +}; +like( $@, qr/Mandatory parameter/, "no proxy" ); + +Test::Nebulous->setup; + +eval { + Nebulous::Client->new( + proxy => "foo", + dog => "do" + ); +}; +like( $@, qr/not listed in the validation options/, "bad param" ); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous/t/50_client_create.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/50_client_create.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous/t/50_client_create.t (revision 24557) @@ -0,0 +1,100 @@ +#!/usr/bin/perl + +# Copryight (C) 2004-2005 Joshua Hoblitt +# +# $Id: 51_client_create.t,v 1.6 2008-05-15 03:24:58 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use Apache::Test qw( -withtestmore ); + +plan tests => 9; + +use lib qw( ./t ./lib ); + +use Nebulous::Client; +use Test::Nebulous; +use Test::URI; +use URI::Split qw( uri_split ); + +my $hostport = Apache::Test->config->{ 'hostport' }; + +Test::Nebulous->setup; + +{ + # key + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + my $uri = $neb->create("foo"); + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "good filename"); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # key, volume + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + my $uri = $neb->create("foo", "node01"); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "good filename"); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + # key, volume == undef + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + my $uri = $neb->create("foo", undef); + + my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); + ok(-e $path, "good filename"); + uri_scheme_ok($uri, 'file'); +} + +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + $neb->create("foo"); + is($neb->create("foo"), undef, "object already exists"); +} + +Test::Nebulous->setup; + +eval { + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + $neb->create(); +}; +like($@, qr/1 - 2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + + $neb->create(1, 2, 3); +}; +like($@, qr/1 - 2 were expected/, "too many params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous/t/50_client_new.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/50_client_new.t (revision 24529) +++ (revision ) @@ -1,60 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 50_client_new.t,v 1.1 2005-12-03 02:52:31 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Apache::Test qw( -withtestmore ); - -plan tests => 4; - -use lib qw( ./t ./lib ); - -use Nebulous::Client; -use Nebulous::Util qw( :standard ); -use Test::Nebulous; - -my $hostport = Apache::Test->config->{ 'hostport' }; - -Test::Nebulous->setup; - -{ - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - isa_ok( $neb, "Nebulous::Client" ); -} - -Test::Nebulous->setup; - -{ - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - uri => "http://example.com/PS/IPP/Nebulous/Client", - ); - - isa_ok( $neb, "Nebulous::Client" ); -} - -Test::Nebulous->setup; - -eval { - Nebulous::Client->new; -}; -like( $@, qr/Mandatory parameter/, "no proxy" ); - -Test::Nebulous->setup; - -eval { - Nebulous::Client->new( - proxy => "foo", - dog => "do" - ); -}; -like( $@, qr/not listed in the validation options/, "bad param" ); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous/t/51_client_create.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/51_client_create.t (revision 24529) +++ (revision ) @@ -1,100 +1,0 @@ -#!/usr/bin/perl - -# Copryight (C) 2004-2005 Joshua Hoblitt -# -# $Id: 51_client_create.t,v 1.6 2008-05-15 03:24:58 jhoblitt Exp $ - -use strict; -use warnings FATAL => qw( all ); - -use Apache::Test qw( -withtestmore ); - -plan tests => 9; - -use lib qw( ./t ./lib ); - -use Nebulous::Client; -use Test::Nebulous; -use Test::URI; -use URI::Split qw( uri_split ); - -my $hostport = Apache::Test->config->{ 'hostport' }; - -Test::Nebulous->setup; - -{ - # key - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - my $uri = $neb->create("foo"); - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "good filename"); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # key, volume - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - my $uri = $neb->create("foo", "node01"); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "good filename"); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - # key, volume == undef - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - my $uri = $neb->create("foo", undef); - - my ($scheme, $auth, $path, $query, $frag) = uri_split($uri); - ok(-e $path, "good filename"); - uri_scheme_ok($uri, 'file'); -} - -Test::Nebulous->setup; - -{ - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create("foo"); - is($neb->create("foo"), undef, "object already exists"); -} - -Test::Nebulous->setup; - -eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create(); -}; -like($@, qr/1 - 2 were expected/, "no params"); - -Test::Nebulous->setup; - -eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create(1, 2, 3); -}; -like($@, qr/1 - 2 were expected/, "too many params"); - -Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous/t/59_client_delete.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/59_client_delete.t (revision 24529) +++ branches/eam_branches/20090522/Nebulous/t/59_client_delete.t (revision 24557) @@ -10,5 +10,5 @@ use Apache::Test qw( -withtestmore ); -plan tests => 9; +plan tests => 13; use lib qw( ./t ./lib ); @@ -26,7 +26,9 @@ proxy => "http://$hostport/nebulous", ); - $neb->create( "foo" ); + my $uri = $neb->create( "foo" ); ok( $neb->delete( "foo" ), "delete object" ); + + ok( ! -e _get_file_path($uri), "deleted file" ); my $locations = $neb->find_instances( "foo" ); @@ -76,4 +78,27 @@ } +# force flag +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + $neb->create( "foo" ); + + is( $neb->delete("foo", undef), 1, "force flag false" ); +} + +Test::Nebulous->setup; + +{ + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + $neb->create( "foo" ); + + is( $neb->delete("foo", 1), 1, "force flag false" ); +} + Test::Nebulous->setup; @@ -84,5 +109,5 @@ $neb->delete(); }; -like( $@, qr/1 was expected/, "no params" ); +like( $@, qr/1 - 2 were expected/, "no params" ); Test::Nebulous->setup; @@ -92,7 +117,17 @@ proxy => "http://$hostport/nebulous", ); - $neb->delete( "foo", 2 ); + $neb->delete( "foo", 3); }; -like( $@, qr/1 was expected/, "too many params" ); +like( $@, qr/is boolean/, "force flag not boolean" ); + +Test::Nebulous->setup; + +eval { + my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", + ); + $neb->delete( "foo", 0, 2 ); +}; +like( $@, qr/1 - 2 were expected/, "too many params" ); Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous/t/62_client_delete_instance.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/62_client_delete_instance.t (revision 24529) +++ branches/eam_branches/20090522/Nebulous/t/62_client_delete_instance.t (revision 24557) @@ -10,5 +10,5 @@ use Apache::Test qw( -withtestmore ); -plan tests => 11; +plan tests => 10; use lib qw( ./t ./lib ); @@ -35,6 +35,4 @@ is( $uri, @$locations[0], "delete instance" ); - - ok( ! -e _get_file_path( @$locations[0] ), "deleted file" ); } Index: branches/eam_branches/20090522/Nebulous/t/66_client_xattr.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/66_client_xattr.t (revision 24529) +++ branches/eam_branches/20090522/Nebulous/t/66_client_xattr.t (revision 24557) @@ -1,7 +1,5 @@ #!/usr/bin/perl -# Copryight (C) 2007-2008 Joshua Hoblitt -# -# $Id: 66_client_xattr.t,v 1.1 2008-05-20 00:54:30 jhoblitt Exp $ +# Copryight (C) 2007-2009 Joshua Hoblitt use strict; @@ -9,6 +7,5 @@ use Apache::Test qw( -withtestmore ); -plan tests => 32; - +plan tests => 44; use lib qw( ./t ./lib ); @@ -20,4 +17,8 @@ my $hostport = Apache::Test->config->{ 'hostport' }; +my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", +); + # 1 key / xattr @@ -25,21 +26,17 @@ { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - my $uri = $neb->create('foo'); - ok($neb->setxattr('foo', 'bar', 'baz', 'create'), 'set object xattr'); + ok($neb->setxattr('foo', 'user.bar', 'baz', 'create'), 'set object xattr'); { my $xattrs = $neb->listxattr('foo'); is(scalar @$xattrs, 1, 'number of xattrs'); - is(@$xattrs[0], 'bar', 'xattr name'); - } - - my $value = $neb->getxattr('foo', 'bar'); + is(@$xattrs[0], 'user.bar', 'xattr name'); + } + + my $value = $neb->getxattr('foo', 'user.bar'); is($value, 'baz', 'xattr value'); - ok($neb->removexattr('foo', 'bar'), "remove object xattr"); + ok($neb->removexattr('foo', 'user.bar'), "remove object xattr"); { my $xattrs = $neb->listxattr('foo'); @@ -53,27 +50,23 @@ { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - my $uri = $neb->create('foo'); - ok($neb->setxattr('foo', 'bar', 'baz', 'create'), 'set object xattr'); - ok($neb->setxattr('foo', 'bonk', 'quix', 'create'), 'set object xattr'); + ok($neb->setxattr('foo', 'user.bar', 'baz', 'create'), 'set object xattr'); + ok($neb->setxattr('foo', 'user.bonk', 'quix', 'create'), 'set object xattr'); { my $xattrs = $neb->listxattr('foo'); is(scalar @$xattrs, 2, 'number of xattrs'); - is(@$xattrs[0], 'bar', 'xattr name'); - is(@$xattrs[1], 'bonk', 'xattr name'); - } - - my $value = $neb->getxattr('foo', 'bar'); + is(@$xattrs[0], 'user.bar', 'xattr name'); + is(@$xattrs[1], 'user.bonk', 'xattr name'); + } + + my $value = $neb->getxattr('foo', 'user.bar'); is($value, 'baz', 'xattr value'); - $value = $neb->getxattr('foo', 'bonk'); + $value = $neb->getxattr('foo', 'user.bonk'); is($value, 'quix', 'xattr value'); - ok($neb->removexattr('foo', 'bar'), "remove object xattr"); - ok($neb->removexattr('foo', 'bonk'), "remove object xattr"); + ok($neb->removexattr('foo', 'user.bar'), "remove object xattr"); + ok($neb->removexattr('foo', 'user.bonk'), "remove object xattr"); { my $xattrs = $neb->listxattr('foo'); @@ -87,23 +80,19 @@ { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - my $uri = $neb->create('foo'); - ok($neb->setxattr('foo', 'bar', 'baz', 'create'), 'set object xattr'); - ok($neb->setxattr('foo', 'bar', 'quix', 'replace'), 're-set object xattr'); + ok($neb->setxattr('foo', 'user.bar', 'baz', 'create'), 'set object xattr'); + ok($neb->setxattr('foo', 'user.bar', 'quix', 'replace'), 're-set object xattr'); { my $xattrs = $neb->listxattr('foo'); is(scalar @$xattrs, 1, 'number of xattrs'); - is(@$xattrs[0], 'bar', 'xattr name'); - } - - my $value = $neb->getxattr('foo', 'bar'); + is(@$xattrs[0], 'user.bar', 'xattr name'); + } + + my $value = $neb->getxattr('foo', 'user.bar'); is($value, 'quix', 'xattr value'); - ok($neb->removexattr('foo', 'bar'), "remove object xattr"); + ok($neb->removexattr('foo', 'user.bar'), "remove object xattr"); { my $xattrs = $neb->listxattr('foo'); @@ -117,8 +106,27 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - + $neb->setxattr('foo', 'user.bar', 'baz', 'create'); +}; +like($@, qr/is valid object key/, "create xattr on non-existant key"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + + $neb->setxattr('foo', 'luser.bar', 'baz', 'create'); +}; +like($@, qr/xattr is in user. namespace/, "user. namspace"); + +Test::Nebulous->setup; + +eval { + $neb->setxattr('foo', 'user.bar', 'baz', 'replace'); +}; +like($@, qr/is valid object key/, "replace xattr on non-existant key"); + +Test::Nebulous->setup; + +eval { $neb->setxattr(); }; @@ -128,11 +136,7 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create('foo'); - - $neb->setxattr('foo', 'bar'); + $neb->create('foo'); + + $neb->setxattr('foo', 'user.bar'); }; like($@, qr/4 were expected/, "too few params"); @@ -141,11 +145,7 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create('foo'); - - $neb->setxattr('foo', 'bar', 'baz'); + $neb->create('foo'); + + $neb->setxattr('foo', 'user.bar', 'baz'); }; like($@, qr/4 were expected/, "too few params"); @@ -154,14 +154,62 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create('foo'); - - $neb->setxattr('foo', 'bar', 'baz', 'create', 'quix'); + $neb->create('foo'); + + $neb->setxattr('foo', 'user.bar', 'baz', 'create', 'quix'); }; like($@, qr/4 were expected/, "too many params"); +# getxattr + +Test::Nebulous->setup; + +eval { + $neb->getxattr('foo', 'bar'); +}; +like($@, qr/is valid object key/, "get xattr from non-existant nebulous key"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + + $neb->getxattr('foo', 'luser.bar'); +}; +like($@, qr/xattr is in user. namespace/, "user. namespace"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + $neb->getxattr('foo', 'user.bar'); +}; +like($@, qr|xattr neb:///foo:user.bar does not exist|, + "get xattr from non-existant xattr key"); + +Test::Nebulous->setup; + +eval { + $neb->getxattr(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + + $neb->getxattr('foo'); +}; +like($@, qr/2 were expected/, "too few params"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + + $neb->getxattr('foo', 'user.bar', 'baz'); +}; +like($@, qr/2 were expected/, "too many params"); + # listxattr @@ -169,8 +217,4 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - $neb->create('foo'); @@ -182,8 +226,4 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - $neb->create('foo'); @@ -192,14 +232,35 @@ like($@, qr/1 was expected/, "too many params"); -# getxattr - -Test::Nebulous->setup; - -eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->getxattr(); +# removexattr + +Test::Nebulous->setup; + +eval { + $neb->removexattr('foo', 'user.bar'); +}; +like($@, qr/is valid object key/, "remove xattr from non-existant nebulous key"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + + $neb->getxattr('foo', 'luser.bar'); +}; +like($@, qr/xattr is in user. namespace/, "user. namespace"); + +Test::Nebulous->setup; + +eval { + $neb->create('foo'); + $neb->removexattr('foo', 'user.bar'); +}; +like($@, qr|xattr neb:///foo:user.bar does not exist|, + "remove xattr from non-existant xattr key"); + +Test::Nebulous->setup; + +eval { + $neb->removexattr(); }; like($@, qr/2 were expected/, "no params"); @@ -208,11 +269,7 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create('foo'); - - $neb->getxattr('foo'); + $neb->create('foo'); + + $neb->removexattr('foo'); }; like($@, qr/2 were expected/, "too few params"); @@ -221,11 +278,7 @@ eval { - my $neb = Nebulous::Client->new( - proxy => "http://$hostport/nebulous", - ); - - $neb->create('foo'); - - $neb->getxattr('foo', 'bar', 'baz'); + $neb->create('foo'); + + $neb->removexattr('foo', 'user.bar', 'baz'); }; like($@, qr/2 were expected/, "too many params"); Index: branches/eam_branches/20090522/Nebulous/t/68_client_chmod.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/68_client_chmod.t (revision 24557) +++ branches/eam_branches/20090522/Nebulous/t/68_client_chmod.t (revision 24557) @@ -0,0 +1,117 @@ +#!/usr/bin/perl + +# Copryight (C) 2009 Joshua Hoblitt + +use strict; +use warnings FATAL => qw( all ); + +use Apache::Test qw( -withtestmore ); + +plan tests => 13; + +use lib qw( ./t ./lib ); + +use Nebulous::Client; +use Nebulous::Util qw( :standard ); +use Test::URI; +use Test::Nebulous; +use URI; +use File::stat; + +my $hostport = Apache::Test->config->{ 'hostport' }; + +my $neb = Nebulous::Client->new( + proxy => "http://$hostport/nebulous", +); + +Test::Nebulous->setup; + +# object with one instance +{ + my $key = "foo"; + my $uri = $neb->create($key); + + my $path = URI->new($uri)->path; + my $st1 = stat($path); + + my $mode = $neb->chmod($key, 0440); + my $st2 = stat($path); + + is($st2->mode &07777, 0440, "chmod() single instance"); + is($mode, 0440, "returned mode"); + is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode"); +} + +Test::Nebulous->setup; + +# object with two instances +{ + my $key = "foo"; + my $uri1 = $neb->create($key); + my $uri2 = $neb->replicate($key); + + my $mode = $neb->chmod($key, 0440); + + my $st1 = stat(URI->new($uri1)->path); + my $st2 = stat(URI->new($uri2)->path); + + is($st1->mode &07777, 0440, "chmod() first instance"); + is($st2->mode &07777, 0440, "chmod() second instance"); + is($mode, 0440, "returned mode"); + is($neb->getxattr($key, 'user.mode'), 0440, "xattr user.mode"); +} + +Test::Nebulous->setup; + +{ + my $mode = $neb->chmod("foo", 0644); + + is($mode, undef, "object does not exist"); +} + +Test::Nebulous->setup; + +{ + $neb->create("foo"); + + my $mode = $neb->chmod("foo", 0640); + + is($mode, undef, "mode is not 0440"); +} + +Test::Nebulous->setup; + +{ + $neb->create("foo"); + + my $mode = $neb->chmod("foo", 999); + + is($mode, undef, "bad mode"); +} + +Test::Nebulous->setup; + +eval { + $neb->chmod(); +}; +like($@, qr/2 were expected/, "no params"); + +Test::Nebulous->setup; + +eval { + $neb->create("foo"); + + $neb->chmod("foo"); +}; +like($@, qr/2 were expected/, "one param"); + +Test::Nebulous->setup; + +eval { + $neb->create("foo"); + + $neb->chmod("foo", 0440, 2); +}; +like($@, qr/2 were expected/, "three params"); + +Test::Nebulous->cleanup; Index: branches/eam_branches/20090522/Nebulous/t/70_neb-ls.t =================================================================== --- branches/eam_branches/20090522/Nebulous/t/70_neb-ls.t (revision 24529) +++ branches/eam_branches/20090522/Nebulous/t/70_neb-ls.t (revision 24557) @@ -38,4 +38,5 @@ # NEB_SERVER env var not set +undef $ENV{'NEB_SERVER'} if defined $ENV{'NEB_SERVER'}; Test::Nebulous->setup; @@ -82,5 +83,5 @@ is($? >> 8, 0, "exit code"); - like($test->stdout, qr/^foo bar$/, "stdout"); + like($test->stdout, qr/^foo\nbar$/, "stdout"); like($test->stderr, qr/^$/, "stderr"); } @@ -160,5 +161,5 @@ is($? >> 8, 0, "exit code"); - like($test->stdout, qr|^a/foo a/bar$|, "stdout"); + like($test->stdout, qr|^a/foo\na/bar$|, "stdout"); like($test->stderr, qr/^$/, "stderr"); } Index: branches/eam_branches/20090522/Nebulous/t/conf/startup.pl.in =================================================================== --- branches/eam_branches/20090522/Nebulous/t/conf/startup.pl.in (revision 24529) +++ branches/eam_branches/20090522/Nebulous/t/conf/startup.pl.in (revision 24557) @@ -18,4 +18,5 @@ my $config = Nebulous::Server::Config->new( trace => 'all', + memcached_servers => ['127.0.0.1:11211'], ); $config->add_db( Index: branches/eam_branches/20090522/Ohana/src/addstar/include/addstar.h =================================================================== --- branches/eam_branches/20090522/Ohana/src/addstar/include/addstar.h (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/addstar/include/addstar.h (revision 24557) @@ -263,4 +263,6 @@ Stars *Convert_PS1_DEV_1 PROTO((FTable *table, int *nstars)); Stars *Convert_PS1_V1 PROTO((FTable *table, int *nstars)); +Stars *Convert_PS1_V1_Alt PROTO((FTable *table, int *nstars)); +Stars *Convert_PS1_V2 PROTO((FTable *table, int *nstars)); int InitStar PROTO((Stars *star)); Index: branches/eam_branches/20090522/Ohana/src/addstar/src/MatchHeaders.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/addstar/src/MatchHeaders.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/addstar/src/MatchHeaders.c (revision 24557) @@ -52,4 +52,5 @@ if (!strcmp (exttype, "PS1_DEV_1")) goto keep; if (!strcmp (exttype, "PS1_V1")) goto keep; + if (!strcmp (exttype, "PS1_V2")) goto keep; continue; Index: branches/eam_branches/20090522/Ohana/src/addstar/src/ReadStarsFITS.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/addstar/src/ReadStarsFITS.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/addstar/src/ReadStarsFITS.c (revision 24557) @@ -43,4 +43,7 @@ stars = Convert_PS1_V1 (&table, &Nstars); } + if (!strcmp (type, "PS1_V2")) { + stars = Convert_PS1_V2 (&table, &Nstars); + } if (stars == NULL) { fprintf (stderr, "ERROR: invalid table type %s\n", type); @@ -188,4 +191,17 @@ Stars *stars; CMF_PS1_V1 *ps1data; + + // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec. this was a + // mistake in two ways: a few format should have been defined (eg, CMF_PS1_V2), and the + // layout used did not have clean byte-boundaries for the corresponding structure. The + // former means we have two varieties of CMF_PS1_V1 out there; the latter means that the + // autocode tools do not work to read in the new version, even if we recognize it. Here we + // test for the existence of the broken version (table[0].headers[0].Naxis[0] == 136), and + // call a special conversion function if it is found. + + if (table[0].header[0].Naxis[0] == 136) { + stars = Convert_PS1_V1_Alt (table, nstars); + return (stars); + } ps1data = gfits_table_get_CMF_PS1_V1 (table, &Nstars, NULL); @@ -244,2 +260,136 @@ return (stars); } + +Stars *Convert_PS1_V1_Alt (FTable *table, int *nstars) { + + int i, Nstars; + double ZeroPt; + Stars *stars; + CMF_PS1_V2 *ps1data; + + // CMF_PS1_V1 was modified 2009.05.26 (r24251) to use doubles for ra & dec. this was a + // mistake in two ways: a few format should have been defined (eg, CMF_PS1_V2), and the + // layout used did not have clean byte-boundaries for the corresponding structure. The + // former means we have two varieties of CMF_PS1_V1 out there; the latter means that the + // autocode tools do not work to read in the new version, even if we recognize it. Here we + // test for the existence of the broken version (table[0].headers[0].Naxis[0] == 136), and + // call a special conversion function if it is found. + + ps1data = gfits_table_get_CMF_PS1_V1_Alt (table, &Nstars, NULL); + ZeroPt = GetZeroPoint(); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ShortDegree(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQual = ps1data[i].psfQual; + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.crNsigma = ps1data[i].crNsigma; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ShortPixels(ps1data[i].fy); + stars[i].measure.theta = ShortDegree(ps1data[i].df); + + stars[i].measure.Mxx = ShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID, + // averef is set in find_matches, dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} + +Stars *Convert_PS1_V2 (FTable *table, int *nstars) { + + int i, Nstars; + double ZeroPt; + Stars *stars; + CMF_PS1_V2 *ps1data; + + ps1data = gfits_table_get_CMF_PS1_V2 (table, &Nstars, NULL); + ZeroPt = GetZeroPoint(); + + ALLOCATE (stars, Stars, Nstars); + for (i = 0; i < Nstars; i++) { + InitStar (&stars[i]); + stars[i].measure.Xccd = ps1data[i].X; + stars[i].measure.Yccd = ps1data[i].Y; + stars[i].measure.dXccd = ShortPixels(ps1data[i].dX); + stars[i].measure.dYccd = ShortPixels(ps1data[i].dY); + + stars[i].measure.posangle = ShortDegree(ps1data[i].posangle); + stars[i].measure.pltscale = ps1data[i].pltscale; + + if ((ps1data[i].M >= 0.0) || isnan(ps1data[i].M)) { + stars[i].measure.M = NAN; + } else { + stars[i].measure.M = ps1data[i].M + ZeroPt; + } + stars[i].measure.dM = ps1data[i].dM; + stars[i].measure.dMcal = ps1data[i].dMcal; + stars[i].measure.Map = ps1data[i].Map + ZeroPt; + + stars[i].measure.Sky = ps1data[i].sky; + stars[i].measure.dSky = ps1data[i].dSky; + + stars[i].measure.psfChisq = ps1data[i].psfChisq; + stars[i].measure.psfQual = ps1data[i].psfQual; + stars[i].measure.psfNdof = ps1data[i].psfNdof; + stars[i].measure.psfNpix = ps1data[i].psfNpix; + stars[i].measure.crNsigma = ps1data[i].crNsigma; + stars[i].measure.extNsigma = ps1data[i].extNsigma; + + stars[i].measure.FWx = ShortPixels(ps1data[i].fx); + stars[i].measure.FWy = ShortPixels(ps1data[i].fy); + stars[i].measure.theta = ShortDegree(ps1data[i].df); + + stars[i].measure.Mxx = ShortPixels(ps1data[i].Mxx); + stars[i].measure.Mxy = ShortPixels(ps1data[i].Mxy); + stars[i].measure.Myy = ShortPixels(ps1data[i].Myy); + + stars[i].measure.photFlags = ps1data[i].flags; + + // this is may optionally be replaced by the internal sequence (see FilterStars.c) + stars[i].measure.detID = ps1data[i].detID; + + // the Average fields and the following Measure fields are set in FilterStars after + // the image metadata is in hand: dR, dD, Mcal, dt, airmass, az, t, imageID, extID, + // averef is set in find_matches, dbFlags is zero on ingest. + + // the following fields are currently not being set anywhere: t_msec + } + *nstars = Nstars; + return (stars); +} Index: branches/eam_branches/20090522/Ohana/src/getstar/src/ReadImageFiles.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/getstar/src/ReadImageFiles.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/getstar/src/ReadImageFiles.c (revision 24557) @@ -136,5 +136,6 @@ !strcmp (tmpword, "PS1_DEV_0") || !strcmp (tmpword, "PS1_DEV_1") || - !strcmp (tmpword, "PS1_V1")) { + !strcmp (tmpword, "PS1_V1") || + !strcmp (tmpword, "PS1_V2")) { exttype[Nimage] = strcreate (tmpword); Index: branches/eam_branches/20090522/Ohana/src/libautocode/Makefile.Targets =================================================================== --- branches/eam_branches/20090522/Ohana/src/libautocode/Makefile.Targets (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/libautocode/Makefile.Targets (revision 24557) @@ -48,4 +48,5 @@ $(ASRC)/cmf-ps1-dev-1.$(ARCH).o \ $(ASRC)/cmf-ps1-v1.$(ARCH).o \ +$(ASRC)/cmf-ps1-v2.$(ARCH).o \ $(ASRC)/cmf-smpdata.$(ARCH).o \ $(ASRC)/getstar-ps1-dev-0.$(ARCH).o \ @@ -107,4 +108,5 @@ $(AINC)/cmf-ps1-dev-1.h \ $(AINC)/cmf-ps1-v1.h \ +$(AINC)/cmf-ps1-v2.h \ $(AINC)/cmf-smpdata.h \ $(AINC)/getstar-ps1-dev-0.h \ Index: branches/eam_branches/20090522/Ohana/src/libautocode/def/autocode.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/libautocode/def/autocode.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/libautocode/def/autocode.c (revision 24557) @@ -7,7 +7,6 @@ if (size != $SIZE) { - fprintf (stderr, "ERROR: mismatch in data types $STRUCT: %d vs %d\n", - size, $SIZE); - exit (1); + fprintf (stderr, "WARNING: mismatch in data types $STRUCT: %d vs %d\n", size, $SIZE); + return (FALSE); } @@ -26,14 +25,50 @@ } +$STRUCT *gfits_downsize_and_convert_$STRUCT ($STRUCT *data, int size, int nitems) { + + int i; + unsigned char *byte, tmp; + $STRUCT *output; + + if ($SIZE > size) { + fprintf (stderr, "ERROR: uncorrectable mismatch in data types $STRUCT: %d vs %d\n", size, $SIZE); + exit (1); + } + + // allocate a new array + ALLOCATE (output, $STRUCT, nitems); + for (i = 0; i < nitems; i++) { + memcpy (&output[i], &data[i], $SIZE); + } + + /* provide initial values to avoid compiler warnings for non-BYTE_SWAP arch */ + i = tmp = 0; + byte = NULL; + +# ifdef BYTE_SWAP + byte = (unsigned char *) output; + for (i = 0; i < nitems; i++, byte += size) { + /** BYTE SWAP **/ + } +# endif + + return (output); +} + /*** add test of EXTNAME and header-defined columns? ***/ /* return internal structure representation */ $STRUCT *gfits_table_get_$STRUCT (FTable *ftable, int *Ndata, int *swapped) { - $STRUCT *data; + $STRUCT *data, *output; *Ndata = ftable[0].header[0].Naxis[1]; data = ($STRUCT *) ftable[0].buffer; if ((swapped == NULL) || (*swapped == FALSE)) { - gfits_convert_$STRUCT (data, sizeof ($STRUCT), *Ndata); + if (!gfits_convert_$STRUCT (data, sizeof ($STRUCT), *Ndata)) { + output = gfits_downsize_and_convert_$STRUCT (data, sizeof ($STRUCT), *Ndata); + free (ftable[0].buffer); + ftable[0].buffer = output; + // XXX do I need to change NX? + } gfits_table_scale_data (ftable); if (swapped != NULL) *swapped = TRUE; Index: branches/eam_branches/20090522/Ohana/src/libautocode/def/cmf-ps1-v2.d =================================================================== --- branches/eam_branches/20090522/Ohana/src/libautocode/def/cmf-ps1-v2.d (revision 24557) +++ branches/eam_branches/20090522/Ohana/src/libautocode/def/cmf-ps1-v2.d (revision 24557) @@ -0,0 +1,78 @@ +# name of structure type +STRUCT CMF_PS1_V2 +EXTNAME CMF_PS1_V2 +TYPE BINTABLE +SIZE 136 + +# elements of data structure / FITS table +FIELD detID, IPP_IDET, unsigned int, detection ID +FIELD X, X_PSF, float, x coord, pixels +FIELD Y, Y_PSF, float, y coord, pixels +FIELD dX, X_PSF_SIG, float, x coord error, pixels +FIELD dY, Y_PSF_SIG, float, y coord error, pixels +FIELD posangle, POSANGLE, float, Posangle at source, degrees +FIELD pltscale, PLTSCALE, float, Plate Scale at source, arcsec/pixel +FIELD M, PSF_INST_MAG, float, inst mags, mags +FIELD dM, PSF_INST_MAG_SIG, float, inst mag error, mags +FIELD Map, AP_MAG_STANDARD, float, standard aperture mag, mags +FIELD apRadius, AP_MAG_RADIUS, float, radius used for fit, pixels +FIELD Mpeak, PEAK_FLUX_AS_MAG, float, peak flux as a mag, mags +FIELD Mcalib, CAL_PSF_MAG, float, calibrated psf mag, mags +FIELD dMcal, CAL_PSF_MAG_SIG, float, zero point scatter, mags +FIELD RA, RA_PSF, double, PSF RA coord, degrees +FIELD DEC, DEC_PSF, double, PSF DEC coord, degrees +FIELD sky, SKY, float, sky flux, cnts/sec +FIELD dSky, SKY_SIG, float, sky flux error, cnts/sec +FIELD psfChisq, PSF_CHISQ, float, psf fit chisq +FIELD crNsigma, CR_NSIGMA, float, Nsigma deviations from PSF to CF +FIELD extNsigma, EXT_NSIGMA, float, Nsigma deviations from PSF to EXT +FIELD fx, PSF_MAJOR, float, psf fit major axis, pixels +FIELD fy, PSF_MINOR, float, psf fit minor axis, pixels +FIELD df, PSF_THETA, float, ellipse angle, degrees +FIELD psfQual, PSF_QF, float, quality factor +FIELD psfNdof, PSF_NDOF, int, psf degrees of freedom +FIELD psfNpix, PSF_NPIX, int, psf number of pixels +FIELD Mxx, MOMENTS_XX, float, second moment X, pixels^2 +FIELD Mxy, MOMENTS_XY, float, second moment Y, pixels^2 +FIELD Myy, MOMENTS_YY, float, second moment XY, pixels^2 +FIELD flags, FLAGS, int, analysis flags +FIELD nFrames, N_FRAMES, short, images overlapping peak +FIELD padding, PADDING, short, padding for 8byte records + +# for an object in an image, we have three triplets that tell us about the shape: +# second moments: Mxx, Mxy, Myy +# model shape parameters: F_major, F_minor, F_theta +# centroid errors: sigma_X, sigma_Y, sigma_XY + +# IPP_IDET 1J IPP_IDET, unsigned i +# X_PSF 1E X_PSF, float, +# Y_PSF 1E Y_PSF, float, +# X_PSF_SIG 1E X_PSF_SIG, float, +# Y_PSF_SIG 1E Y_PSF_SIG, float, +# RA_PSF 1E RA_PSF, float, +# DEC_PSF 1E DEC_PSF, float, +# POSANGLE 1E POSANGLE, float, +# PLTSCALE 1E PLTSCALE, float, +# PSF_INST_MAG 1E PSF_INST_MAG, float, +# PSF_INST_MAG_SIG 1E PSF_INST_MAG_SIG, float, +# AP_MAG_STANDARD 1E AP_MAG_STANDARD, float, +# AP_MAG_RADIUS 1E +# PEAK_FLUX_AS_MAG 1E PEAK_FLUX_AS_MAG, float, +# CAL_PSF_MAG 1E CAL_PSF_MAG, float, +# CAL_PSF_MAG_SIG 1E CAL_PSF_MAG_SIG, float, +# SKY 1E SKY, float, +# SKY_SIGMA 1E SKY_SIG, float, +# PSF_CHISQ 1E PSF_CHISQ, float, +# CR_NSIGMA 1E CR_NSIGMA, float, +# EXT_NSIGMA 1E EXT_NSIGMA, float, +# PSF_MAJOR 1E PSF_MAJOR, float, +# PSF_MINOR 1E PSF_MINOR, float, +# PSF_THETA 1E PSF_THETA, float, +# PSF_QF 1E PSF_QF, float, +# PSF_NDOF 1J PSF_NDOF, int, +# PSF_NPIX 1J PSF_NPIX, int, +# MOMENTS_XX 1E MOMENTS_XX, float, +# MOMENTS_XY 1E MOMENTS_XY, float, +# MOMENTS_YY 1E MOMENTS_YY, float, +# N_FRAMES 1I N_FRAMES, short, +# FLAGS 1J FLAGS, int, Index: branches/eam_branches/20090522/Ohana/src/libdvo/Makefile =================================================================== --- branches/eam_branches/20090522/Ohana/src/libdvo/Makefile (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/libdvo/Makefile (revision 24557) @@ -69,5 +69,6 @@ $(SRC)/skyregion_io.$(ARCH).o \ $(SRC)/skyregion_gsc.$(ARCH).o \ -$(SRC)/skyregion_ops.$(ARCH).o +$(SRC)/skyregion_ops.$(ARCH).o \ +$(SRC)/cmf-ps1-v1-alt.$(ARCH).o # $(SRC)/dvo_convert_panstarrs.$(ARCH).o \ @@ -118,2 +119,4 @@ cleandef: cd $(AUTO) && make clean + +.PRECIOUS: $(ASRC)/%.c Index: branches/eam_branches/20090522/Ohana/src/libdvo/include/dvo.h =================================================================== --- branches/eam_branches/20090522/Ohana/src/libdvo/include/dvo.h (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/libdvo/include/dvo.h (revision 24557) @@ -108,6 +108,6 @@ # define ID_MEAS_POOR_ASTROM 0x0010 /* detection is astrometry outlier */ # define ID_MEAS_SKIP_ASTROM 0x0020 /* detection was ignored for astrometry measurement */ -# define ID_MEAS_UNDEF_1 0x0040 /* unused */ -# define ID_MEAS_UNDEF_2 0x0080 /* unused */ +# define ID_MEAS_USED_OBJ 0x0040 /* detection was used during opdate objects */ +# define ID_MEAS_USED_CHIP 0x0080 /* detection was used during update chips */ # define ID_MEAS_BLEND_MEAS 0x0100 /* detection is within radius of multiple objects */ # define ID_MEAS_BLEND_OBJ 0x0200 /* multiple detections within radius of object */ @@ -184,4 +184,7 @@ SkyRegion **regions; } SkyList; + +// special-case function: +CMF_PS1_V2 *gfits_table_get_CMF_PS1_V1_Alt (FTable *ftable, int *Ndata, int *swapped); # if (0) Index: branches/eam_branches/20090522/Ohana/src/libdvo/src/cmf-ps1-v1-alt.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/libdvo/src/cmf-ps1-v1-alt.c (revision 24557) +++ branches/eam_branches/20090522/Ohana/src/libdvo/src/cmf-ps1-v1-alt.c (revision 24557) @@ -0,0 +1,126 @@ +# include "dvo.h" + +CMF_PS1_V2 *gfits_table_get_CMF_PS1_V1_Alt (FTable *ftable, int *Ndata, int *swapped) { + + int i, nitems; + unsigned char *byte, *inbyte, *otbyte, tmp; + CMF_PS1_V2 *output; + + /* provide initial values to avoid compiler warnings for non-BYTE_SWAP arch */ + i = tmp = 0; + byte = NULL; + + // this function is a special case : it must have Nx = 136 + if (ftable[0].header[0].Naxis[0] != 136) { + fprintf (stderr, "ERROR: wrong format for CMF_PS1_V1_Alt: %d vs %d\n", ftable[0].header[0].Naxis[0], 136); + exit (2); + } + + *Ndata = ftable[0].header[0].Naxis[1]; + nitems = ftable[0].header[0].Naxis[1]; + + if ((swapped == NULL) || (*swapped == FALSE)) { + +# ifdef BYTE_SWAP + // we need to do the byte swap before applying the table scaling: + byte = (unsigned char *) ftable[0].buffer; + for (i = 0; i < nitems; i++, byte += 136) { + /** BYTE SWAP **/ + SWAP_WORD (0); // IPP_IDET + SWAP_WORD (4); // X_PSF + SWAP_WORD (8); // Y_PSF + SWAP_WORD (12); // X_PSF_SIG + SWAP_WORD (16); // Y_PSF_SIG + SWAP_DBLE (20); // RA_PSF + SWAP_DBLE (28); // DEC_PSF + SWAP_WORD (36); // POSANGLE + SWAP_WORD (40); // PLTSCALE + SWAP_WORD (44); // PSF_INST_MAG + SWAP_WORD (48); // PSF_INST_MAG_SIG + SWAP_WORD (52); // AP_MAG_STANDARD + SWAP_WORD (56); // AP_MAG_RADIUS + SWAP_WORD (60); // PEAK_FLUX_AS_MAG + SWAP_WORD (64); // CAL_PSF_MAG + SWAP_WORD (68); // CAL_PSF_MAG_SIG + SWAP_WORD (72); // SKY + SWAP_WORD (76); // SKY_SIG + SWAP_WORD (80); // PSF_CHISQ + SWAP_WORD (84); // CR_NSIGMA + SWAP_WORD (88); // EXT_NSIGMA + SWAP_WORD (92); // PSF_MAJOR + SWAP_WORD (96); // PSF_MINOR + SWAP_WORD (100); // PSF_THETA + SWAP_WORD (104); // PSF_QF + SWAP_WORD (108); // PSF_NDOF + SWAP_WORD (112); // PSF_NPIX + SWAP_WORD (116); // MOMENTS_XX + SWAP_WORD (120); // MOMENTS_XY + SWAP_WORD (124); // MOMENTS_YY + SWAP_WORD (128); // FLAGS + SWAP_BYTE (132); // N_FRAMES + SWAP_BYTE (134); // PADDING + } +# endif + + gfits_table_scale_data (ftable); + if (swapped != NULL) *swapped = TRUE; + } + + byte = (unsigned char *) ftable[0].buffer; + + // allocate a new output data buffer + ALLOCATE (output, CMF_PS1_V2, nitems); + inbyte = (unsigned char *) byte; + otbyte = (unsigned char *) output; + + // the data in the input table does not line up with the output structure: copy carefully. + for (i = 0; i < nitems; i++, inbyte += 136, otbyte += 136) { + memcpy (&otbyte[0], &inbyte[0], 20); + memcpy (&otbyte[20], &inbyte[36], 36); + memcpy (&otbyte[56], &inbyte[20], 16); + memcpy (&otbyte[72], &inbyte[72], 62); + } + + free (ftable[0].buffer); + ftable[0].buffer = output; + + // XXX other mods to make ftable consistent with CMF_PS1_V2? (Nx, EXTNAME?) + + return (output); +} + +// data organization (input vs output) +// FITS struct +// WORD 0 IPP_IDET 0 IPP_IDET +// WORD 4 X_PSF 4 X_PSF +// WORD 8 Y_PSF 8 Y_PSF +// WORD 12 X_PSF_SIG 12 X_PSF_SIG +// WORD 16 Y_PSF_SIG 16 Y_PSF_SIG +// DBLE 20 RA_PSF 20 POSANGLE +// DBLE 28 DEC_PSF 24 PLTSCALE +// WORD 36 POSANGLE 28 PSF_INST_MAG +// WORD 40 PLTSCALE 32 PSF_INST_MAG_SIG +// WORD 44 PSF_INST_MAG 36 AP_MAG_STANDARD +// WORD 48 PSF_INST_MAG_SIG 40 AP_MAG_RADIUS +// WORD 52 AP_MAG_STANDARD 44 PEAK_FLUX_AS_MAG +// WORD 56 AP_MAG_RADIUS 48 CAL_PSF_MAG +// WORD 60 PEAK_FLUX_AS_MAG 52 CAL_PSF_MAG_SIG +// WORD 64 CAL_PSF_MAG 56 RA_PSF +// WORD 68 CAL_PSF_MAG_SIG 64 DEC_PSF +// WORD 72 SKY 72 SKY +// WORD 76 SKY_SIG 76 SKY_SIG +// WORD 80 PSF_CHISQ 80 PSF_CHISQ +// WORD 84 CR_NSIGMA 84 CR_NSIGMA +// WORD 88 EXT_NSIGMA 88 EXT_NSIGMA +// WORD 92 PSF_MAJOR 92 PSF_MAJOR +// WORD 96 PSF_MINOR 96 PSF_MINOR +// WORD 100 PSF_THETA 100 PSF_THETA +// WORD 104 PSF_QF 104 PSF_QF +// WORD 108 PSF_NDOF 108 PSF_NDOF +// WORD 112 PSF_NPIX 112 PSF_NPIX +// WORD 116 MOMENTS_XX 116 MOMENTS_XX +// WORD 120 MOMENTS_XY 120 MOMENTS_XY +// WORD 124 MOMENTS_YY 124 MOMENTS_YY +// WORD 128 FLAGS 128 FLAGS +// BYTE 132 N_FRAMES 132 N_FRAMES +// BYTE 134 PADDING 134 PADDING Index: branches/eam_branches/20090522/Ohana/src/relastro/Makefile =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/Makefile (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/Makefile (revision 24557) @@ -55,5 +55,6 @@ $(SRC)/relastro_objects.$(ARCH).o \ $(SRC)/save_catalogs.$(ARCH).o \ -$(SRC)/write_coords.$(ARCH).o +$(SRC)/write_coords.$(ARCH).o \ +$(SRC)/relastroVisual.$(ARCH).o $(RELASTRO): $(INC)/relastro.h $(KAPA_INCS) Index: branches/eam_branches/20090522/Ohana/src/relastro/include/relastro.h =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/include/relastro.h (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/include/relastro.h (revision 24557) @@ -44,5 +44,5 @@ double Ro, dRo; double Do, dDo; - + double uR, duR; double uD, duD; @@ -63,5 +63,5 @@ char flags; Coords coords; -} Mosaic; +} Mosaic; typedef struct { @@ -87,4 +87,5 @@ double SIGMA_LIM; +int SRC_MEAS_TOOFEW; //catalog objects wich fewer detections then this are ignored double MIN_ERROR; @@ -119,4 +120,7 @@ int TimeSelect; time_t TSTART, TSTOP; + +int FlagOutlier; +int CLIP_THRESH; FitMode FIT_MODE; @@ -145,5 +149,5 @@ int corner_check PROTO((double *x1, double *y1, double *x2, double *y2)); void dumpGrid PROTO((void)); -void dump_grid PROTO((void)); +void dump_grid PROTO((void)); int edge_check PROTO((double *x1, double *y1, double *x2, double *y2)); void findImages PROTO((Catalog *catalog, int Ncatalog)); @@ -220,5 +224,5 @@ int setMrelOutput PROTO((Catalog *catalog, int Ncatalog, int mark)); void set_ZP PROTO((double ZERO)); -int setrefcode PROTO((Image *image, int Nimage)); +int setrefcode PROTO((Image *image, int Nimage)); void skip_measurements PROTO((Catalog *catalog, int pass)); void sortA PROTO((double *X, int N)); @@ -265,4 +269,6 @@ int UpdateMeasures (Catalog *catalog, int Ncatalog); void fixImageRaw (Catalog *catalog, int Ncatalog, int im); +void FlagOutliers(Catalog *catalog); +int MeasFilterTest(Measure *measure); int sun_ecliptic (double jd, double *lambda, double *beta, double *epsilon); Index: branches/eam_branches/20090522/Ohana/src/relastro/src/ConfigInit.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/ConfigInit.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/ConfigInit.c (revision 24557) @@ -20,4 +20,5 @@ GetConfig (config, "RELASTRO_SIGMA_LIM", "%lf", 0, &SIGMA_LIM); + GetConfig (config, "RELASTRO_SRC_MEAS_TOOFEW", "%d", 0, &SRC_MEAS_TOOFEW); // XXX these are used in relphot to identify poor stars / images -- define @@ -55,5 +56,5 @@ fprintf (stderr, "directory %s does not exist, giving up\n", CATDIR); exit (1); - } + } /* update master photcode table if not defined */ Index: branches/eam_branches/20090522/Ohana/src/relastro/src/FitChip.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/FitChip.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/FitChip.c (revision 24557) @@ -1,3 +1,4 @@ # include "relastro.h" +# include "relastroVisual.h" // XXX make these user parameters @@ -5,4 +6,5 @@ # define FIT_CHIP_NITER 3 # define FIT_CHIP_NSIGMA 3.0 + // XXX we should test if the fit is sufficiently constrained across the chip, or if the @@ -22,5 +24,4 @@ ALLOCATE (values, double, Nmatch); - for (Niter = 0; Niter < FIT_CHIP_NITER; Niter ++) { @@ -28,10 +29,10 @@ for (i = Nscatter = 0; i < Nmatch; i++) { if (raw[i].mask) continue; - if (raw[i].dMag > FIT_CHIP_MAX_ERROR) continue; + if (isnan(raw[i].dMag) || raw[i].dMag > FIT_CHIP_MAX_ERROR) continue; dL = raw[i].L - ref[i].L; dM = raw[i].M - ref[i].M; dR = hypot (dL, dM); - + values[Nscatter] = dR; Nscatter++; @@ -41,4 +42,6 @@ dsort (values, Nscatter); dRmax = FIT_CHIP_NSIGMA*values[(int)(0.40*Nscatter)]; + relastroVisualPlotScatter(values, dRmax, Nscatter); + relastroVisualPlotRawRef(raw, ref, dRmax, Nmatch); // fit the requested order polynomial @@ -50,5 +53,10 @@ // generate the fit matches for (i = 0; i < Nmatch; i++) { - if (raw[i].mask) continue; + if (raw[i].mask) { + continue; + } + if (isnan(raw[i].dMag) || raw[i].dMag > FIT_CHIP_MAX_ERROR) { + continue; + } // only keep objects within dRmax @@ -57,5 +65,5 @@ dR = hypot (dL, dM); if (dR > dRmax) continue; - + fit_add (fit, raw[i].X, raw[i].Y, ref[i].L, ref[i].M, raw[i].dPos); } @@ -66,15 +74,15 @@ case 0: case 1: - skip = (fit[0].Npts < 8); - break; + skip = (fit[0].Npts < 8); + break; case 2: - skip = (fit[0].Npts < 11); - break; + skip = (fit[0].Npts < 11); + break; case 3: - skip = (fit[0].Npts < 15); - break; + skip = (fit[0].Npts < 15); + break; default: - fprintf (stderr, "invalid chip order %d\n", coords[0].Npolyterms); - abort (); + fprintf (stderr, "invalid chip order %d\n", coords[0].Npolyterms); + abort (); } if (skip) { @@ -95,4 +103,5 @@ XY_to_LM (&raw[i].L, &raw[i].M, raw[i].X, raw[i].Y, coords); } + } Index: branches/eam_branches/20090522/Ohana/src/relastro/src/GetAstromError.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/GetAstromError.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/GetAstromError.c (revision 24557) @@ -30,4 +30,8 @@ dPtotal = sqrt(SQ(dPsys) + AS*SQ(dPobs) + MS*SQ(dM)); + + //XXX dXccd, dYccd are now working correctly + //dPtotal = AS * dPobs; + dPtotal = MAX (dPtotal, MIN_ERROR); Index: branches/eam_branches/20090522/Ohana/src/relastro/src/ImageOps.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/ImageOps.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/ImageOps.c (revision 24557) @@ -1,3 +1,4 @@ # include "relastro.h" +#define TESTING static unsigned int *start; @@ -91,7 +92,7 @@ for (i = 0; VERBOSE && (i < Nimage); i++) { name = GetPhotcodeNamebyCode (image[i].photcode); - fprintf (stderr, "image %d has %d measures (%s, %s)\n", i, Nlist[i], - ohana_sec_to_date(image[i].tzero), name); - } + fprintf (stderr, "image %d has %d measures (%s, %s)\n", i, Nlist[i], + ohana_sec_to_date(image[i].tzero), name); + } } @@ -101,5 +102,5 @@ int i; Measure *measure; - + measure = &catalog[cat].measure[meas]; @@ -110,5 +111,5 @@ if (measure[0].t < start[i]) continue; if (measure[0].t > stop[i]) continue; - + // index for (catalog, measure) -> image bin[cat][meas] = i; @@ -125,5 +126,5 @@ REALLOCATE (clist[i], int, NLIST[i]); REALLOCATE (mlist[i], int, NLIST[i]); - } + } return; } @@ -157,5 +158,5 @@ plot_defaults (&graphdata); - graphdata.ymin = PlotdMmin; + graphdata.ymin = PlotdMmin; graphdata.ymax = PlotdMmax; plot_list (&graphdata, xlist, Mlist, Nimage, "airmass vs Mcal", "airmass.png"); @@ -163,5 +164,5 @@ plot_list (&graphdata, Mlist, dlist, Nimage, "Mcal vs dMcal", NULL); -# define NBIN 200 +# define NBIN 20 REALLOCATE (xlist, double, NBIN); REALLOCATE (Mlist, double, NBIN); @@ -191,8 +192,8 @@ int i, m, c, n; double X, Y, L, M, P, Q, R, D, dR, dD; - + Mosaic *mosaic; Coords *moscoords, *imcoords; - + moscoords = NULL; mosaic = getMosaicForImage (im); @@ -224,5 +225,5 @@ dR = 3600.0*(catalog[c].average[n].R - R); dD = 3600.0*(catalog[c].average[n].D - D); - + if (fabs(catalog[c].measure[m].dR - dR) > 10.0) { // XXXXX running into this still for last megacam exposure: wrong mosaic? @@ -249,5 +250,5 @@ catalog[c].measure[m].dR = 3600.0*(catalog[c].average[n].R - R); } - } + } return; } @@ -259,9 +260,9 @@ int i, m, c, n; - + Mosaic *mosaic; Coords *moscoords; StarData *raw; - + ALLOCATE (raw, StarData, Nlist[im]); @@ -271,6 +272,6 @@ mosaic = getMosaicForImage (im); if (mosaic == NULL) { - fprintf (stderr, "mosaic not found for image %s\n", image[im].name); - exit (1); + fprintf (stderr, "mosaic not found for image %s\n", image[im].name); + exit (1); } moscoords = &mosaic[0].coords; @@ -284,5 +285,5 @@ raw[i].X = catalog[c].measure[m].Xccd; raw[i].Y = catalog[c].measure[m].Yccd; - + raw[i].Mag = catalog[c].measure[m].M; raw[i].dMag = catalog[c].measure[m].dM; @@ -292,27 +293,35 @@ // an object with only one detection provides no information about the image calibration + //XXX this is already taken care of in bcatalog raw[i].mask = FALSE; - if (catalog[c].average[n].Nmeasure < 2) { - raw[i].mask = TRUE; - } + int mask = FALSE; + if (catalog[c].average[n].Nmeasure <= SRC_MEAS_TOOFEW) { + mask = TRUE; + } + if (!finite(catalog[c].measure[m].dR) || !finite(catalog[c].measure[m].dD)) { + mask = TRUE; + } + + raw[i].mask = mask; + switch (mode) { case MODE_SIMPLE: - /* note that for a Simple image, L,M = P,Q */ - XY_to_LM (&raw[i].L, &raw[i].M, raw[i].X, raw[i].Y, &image[im].coords); - raw[i].P = raw[i].L; - raw[i].Q = raw[i].M; - LM_to_RD (&raw[i].R, &raw[i].D, raw[i].P, raw[i].Q, &image[im].coords); - break; + /* note that for a Simple image, L,M = P,Q */ + XY_to_LM (&raw[i].L, &raw[i].M, raw[i].X, raw[i].Y, &image[im].coords); + raw[i].P = raw[i].L; + raw[i].Q = raw[i].M; + LM_to_RD (&raw[i].R, &raw[i].D, raw[i].P, raw[i].Q, &image[im].coords); + break; case MODE_MOSAIC: - XY_to_LM (&raw[i].L, &raw[i].M, raw[i].X, raw[i].Y, &image[im].coords); - XY_to_LM (&raw[i].P, &raw[i].Q, raw[i].L, raw[i].M, moscoords); - LM_to_RD (&raw[i].R, &raw[i].D, raw[i].P, raw[i].Q, moscoords); - break; + XY_to_LM (&raw[i].L, &raw[i].M, raw[i].X, raw[i].Y, &image[im].coords); + XY_to_LM (&raw[i].P, &raw[i].Q, raw[i].L, raw[i].M, moscoords); + LM_to_RD (&raw[i].R, &raw[i].D, raw[i].P, raw[i].Q, moscoords); + break; default: fprintf (stderr, "error: invalid mode in getImageRaw"); abort (); } - } + } *Nstars = Nlist[im]; @@ -330,5 +339,5 @@ Coords *moscoords; StarData *ref; - + ALLOCATE (ref, StarData, Nlist[im]); @@ -352,5 +361,5 @@ ref[i].R = catalog[c].average[n].R; ref[i].D = catalog[c].average[n].D; - + ref[i].Mag = catalog[c].measure[m].M; ref[i].dMag = catalog[c].measure[m].dM; @@ -373,10 +382,294 @@ break; default: - fprintf (stderr, "invalid case"); - abort(); - } - } - + fprintf (stderr, "invalid case"); + abort(); + } + } + *Nstars = Nlist[im]; return (ref); } + +/** lifted from relphot/StarOps.clean_measures */ +void FlagOutliers2D(Catalog *catalog); + +void FlagOutliers (Catalog *catalog) { + // XXX FlagOutliers is just using FlagOutliers2D + FlagOutliers2D(catalog); + return; + + int i, j, k, m, N, Ndel, Nave, Nmax, TOOFEW, Nsecfilt; + double Ns, theta, x, y; + double *R, *D, *dR, *dD; + StatType statsR, statsD; + + Nsecfilt = GetPhotcodeNsecfilt(); + assert(catalog[0].Nsecfilt == Nsecfilt); + + if (VERBOSE) fprintf (stderr, "marking poor measures\n"); + Nmax = 0; + for (i = 0; i < catalog[0].Naverage; i++) { + Nmax = MAX (Nmax, catalog[0].average[i].Nmeasure); + } + + ALLOCATE (R, double, Nmax); + ALLOCATE (D, double, Nmax); + ALLOCATE (dR, double, Nmax); + ALLOCATE (dD, double, Nmax); + + /* it makes no sense to mark 3-sigma outliers with <5 measurements */ + TOOFEW = MAX (5, SRC_MEAS_TOOFEW); + + Ns = CLIP_THRESH; + Ndel = Nave = 0; + + /* loop over each object in the catalog */ + for (j = 0; j < catalog[0].Naverage; j++) { + + /* accumulate list of valid measurements */ + m = catalog[0].average[j].measureOffset; + N = 0; + for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { + + // skip measurements based on user selected criteria + if (!MeasFilterTest(&catalog[0].measure[m])) continue; + R[N] = catalog[0].measure[m].dR; + D[N] = catalog[0].measure[m].dD; + dR[N] = GetAstromError( &catalog[0].measure[m], ERROR_MODE_RA); + dD[N] = GetAstromError( &catalog[0].measure[m], ERROR_MODE_DEC); + if (isnan(R[N]) || isnan(D[N])) continue; + N++; + } + if (N <= TOOFEW) continue; + + /* 3-sigma clip based on stats of inner 50% */ + initstats ("MEAN"); + liststats (R, dR, N, &statsR); + liststats (D, dD, N, &statsD); + + statsR.sigma = MAX (MIN_ERROR, statsR.sigma); + statsD.sigma = MAX (MIN_ERROR, statsD.sigma); + + /* compare per-object distance to this standard deviation, and flag outliers*/ + m = catalog[0].average[j].measureOffset; + N = 0; + for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { + //reset flag on each invocation + catalog[0].measure[m].dbFlags &= ~ID_MEAS_POOR_ASTROM; + + // skip measurements based on user selected criteria + if (!MeasFilterTest(&catalog[0].measure[m])) continue; + + x = catalog[0].measure[m].dR - statsR.median; + y = catalog[0].measure[m].dD - statsD.median; + theta = atan2(y,x); + if ((x*x + y*y) > (SQR(statsR.sigma * Ns * cos(theta)) + + SQR(statsD.sigma * Ns * sin(theta)))) { + catalog[0].measure[m].dbFlags |= ID_MEAS_POOR_ASTROM; + Ndel++; + } + N++; + Nave ++; + } + + //examine results + relastroVisualPlotOutliers(catalog, catalog[0].average[j].measureOffset, + catalog[0].average[j].Nmeasure, + statsR, statsD, Ns); + } + + if (VERBOSE) fprintf (stderr, "%d measures marked poor, %d total\n", Ndel, Nave); + free (R); + free(dR); + free(D); + free(dD); +} + + +/** an alternative outlier rejection scheme */ +void FlagOutliers2D (Catalog *catalog) { + + int i, j, k, m, N, Ndel, Nave, Nmax, TOOFEW, Nsecfilt; + double *index; + double Ns, theta, x, y; + double *R, *D, *dR, *dD, *d2; + StatType statsR, statsD; + + Nsecfilt = GetPhotcodeNsecfilt(); + assert(catalog[0].Nsecfilt == Nsecfilt); + + if (VERBOSE) fprintf (stderr, "marking poor measures\n"); + Nmax = 0; + for (i = 0; i < catalog[0].Naverage; i++) { + Nmax = MAX (Nmax, catalog[0].average[i].Nmeasure); + } + + ALLOCATE (R, double, Nmax); + ALLOCATE (D, double, Nmax); + ALLOCATE (dR, double, Nmax); + ALLOCATE (dD, double, Nmax); + ALLOCATE (d2, double, Nmax); + ALLOCATE (index, double, Nmax); + + /* it makes no sense to mark 3-sigma outliers with <5 measurements */ + TOOFEW = MAX (5, SRC_MEAS_TOOFEW); + + Ns = CLIP_THRESH; + Ndel = Nave = 0; + + /* loop over each object in the catalog */ + for (j = 0; j < catalog[0].Naverage; j++) { + + /* accumulate list of valid measurements */ + m = catalog[0].average[j].measureOffset; + N = 0; + + for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { + + //reset flag on each invocation + catalog[0].measure[m].dbFlags &= ~ID_MEAS_POOR_ASTROM; + + // skip measurements based on user selected criteria + if (!MeasFilterTest(&catalog[0].measure[m])) continue; + R[N] = catalog[0].measure[m].dR; + D[N] = catalog[0].measure[m].dD; + dR[N] = GetAstromError( &catalog[0].measure[m], ERROR_MODE_RA); + dD[N] = GetAstromError( &catalog[0].measure[m], ERROR_MODE_DEC); + if (isnan(R[N]) || isnan(D[N])) continue; + N++; + } + if (N <= TOOFEW) continue; + + /* calculate mean of all points*/ + initstats ("MEAN"); + liststats (R, dR, N, &statsR); + liststats (D, dD, N, &statsD); + statsR.sigma = MAX (MIN_ERROR, statsR.sigma); + statsD.sigma = MAX (MIN_ERROR, statsD.sigma); + + /* calculate deviations of all points*/ + m = catalog[0].average[j].measureOffset; + N = 0; + for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { + //skip bad measurements + if (!MeasFilterTest(&catalog[0].measure[m])) continue; + x = catalog[0].measure[m].dR - statsR.median; + y = catalog[0].measure[m].dD - statsD.median; + theta = atan2(y,x); + d2[N] = (x*x + y*y) / (SQR(statsR.sigma * Ns * cos(theta)) + + SQR(statsD.sigma * Ns * sin(theta))); + index[N] = m; + N++; + } + + // sort d2 + dsortpair(d2, index, N); + N = (N/2 > (N-1)) ? N/2 : N-1; + + // recalculate image center, sigma based on closest 50% of points + for(k = 0; k < N; k++) { + int ind = (int) index[k]; + R[k] = catalog[0].measure[ind].dR; + D[k] = catalog[0].measure[ind].dD; + dR[k] = GetAstromError( &catalog[0].measure[ind], ERROR_MODE_RA); + dD[k] = GetAstromError( &catalog[0].measure[ind], ERROR_MODE_DEC); + } + liststats (R, dR, N, &statsR); + liststats (D, dD, N, &statsD); + statsR.sigma = MAX (MIN_ERROR, statsR.sigma); + statsD.sigma = MAX (MIN_ERROR, statsD.sigma); + + // use these new statistics to flag outliers + m = catalog[0].average[j].measureOffset; + N = 0; + for (k = 0; k < catalog[0].average[j].Nmeasure; k++, m++) { + //skip bad measurements + if (!MeasFilterTest(&catalog[0].measure[m])) continue; + x = catalog[0].measure[m].dR - statsR.median; + y = catalog[0].measure[m].dD - statsD.median; + theta = atan2(y,x); + d2[N] = (x*x + y*y) / (SQR(statsR.sigma * Ns * cos(theta)) + + SQR(statsD.sigma * Ns * sin(theta))); + if ((d2[N]) > 1) { + catalog[0].measure[m].dbFlags |= ID_MEAS_POOR_ASTROM; + Ndel ++; + } + N++; + Nave++; + } //done rejecting outliers + + //examine results + relastroVisualPlotOutliers(catalog, catalog[0].average[j].measureOffset, + catalog[0].average[j].Nmeasure, + statsR, statsD, Ns); + + } //done looping over objects + + if (VERBOSE) fprintf (stderr, "%d measures marked poor, %d total\n", Ndel, Nave); + free (R); + free(dR); + free(D); + free(dD); + free(d2); + free(index); +} + + + +/** Determine whether a measurement should be included in the analysis, based on supplied filter criteria */ +int MeasFilterTest(Measure *measure) { + int found, k; + long mask; + PhotCode *code; + float mag; + + if (!finite(measure[0].dR) || !finite(measure[0].dD)) return FALSE; + if (!finite(measure[0].M)) return FALSE; //XXX is this necessary for all relastro tasks? + + /* select measurements by photcode, or equiv photcode, if specified */ + if (NphotcodesKeep > 0) { + found = FALSE; + for (k = 0; (k < NphotcodesKeep) && !found; k++) { + if (photcodesKeep[k][0].code == measure[0].photcode) found = TRUE; + if (photcodesKeep[k][0].code == GetPhotcodeEquivCodebyCode(measure[0].photcode)) found = TRUE; + } + if (!found) return FALSE; + } + + if (NphotcodesSkip > 0) { + found = FALSE; + for (k = 0; (k < NphotcodesSkip) && !found; k++) { + if (photcodesSkip[k][0].code == measure[0].photcode) found = TRUE; + if (photcodesSkip[k][0].code == GetPhotcodeEquivCodebyCode(measure[0].photcode)) found = TRUE; + } + if (found) return FALSE; + } + + /* select measurements by time */ + if (TimeSelect) { + if (measure[0].t < TSTART) return FALSE; + if (measure[0].t > TSTOP) return FALSE; + } + + /* select measurements by quality */ + if (PhotFlagSelect) { + if (PhotFlagBad) { + mask = PhotFlagBad; + } else { + code = GetPhotcodebyCode (measure[0].photcode); + mask = code[0].astromBadMask; + } + if (mask & measure[0].photFlags) return FALSE; + } + + /* select measurements by measurement error */ + if ((SIGMA_LIM > 0) && (measure[0].dM > SIGMA_LIM)) return FALSE; + + /* select measurements by mag limit */ + if (ImagSelect) { + mag = PhotInst (measure); + if (mag < ImagMin || mag > ImagMax) return FALSE; + } + + return TRUE; +} Index: branches/eam_branches/20090522/Ohana/src/relastro/src/UpdateObjects.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/UpdateObjects.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/UpdateObjects.c (revision 24557) @@ -97,6 +97,15 @@ for (k = 0; k < catalog[i].average[j].Nmeasure; k++, m++) { - //exclude measurements which have non-finite astrometry - if (!finite(catalog[i].measure[m].dR) || !finite(catalog[i].measure[m].dD)) continue; + //does the measurement pass the supplied filtering constraints? + if (!MeasFilterTest(&catalog[i].measure[m])) { + catalog[i].measure[m].dbFlags &= ~ID_MEAS_USED_OBJ; + continue; + } + + //outlier rejection + if (FlagOutlier && (catalog[i].measure[m].dbFlags & ID_MEAS_POOR_ASTROM)) { + catalog[i].measure[m].dbFlags &= ~ID_MEAS_USED_OBJ; + continue; + } // exclude measurements by previous outlier detection @@ -109,48 +118,5 @@ # endif - /* exclude measurements by quality */ - if (PhotFlagSelect) { - if (PhotFlagBad) { - mask = PhotFlagBad; - } else { - code = GetPhotcodebyCode (catalog[i].measure[m].photcode); - mask = code[0].astromBadMask; - } - if (mask & catalog[i].measure[m].photFlags) { - catalog[i].measure[m].dbFlags |= ID_MEAS_SKIP_ASTROM; - continue; - } - } - - /* exclude measurements by mag limit */ - if (ImagSelect) { - mag = PhotInst (&catalog[i].measure[m]); - if (mag < ImagMin) { - catalog[i].measure[m].dbFlags |= ID_MEAS_SKIP_ASTROM; - continue; - } - if (mag > ImagMax) { - catalog[i].measure[m].dbFlags |= ID_MEAS_SKIP_ASTROM; - continue; - } - } - - /* select or exclude measurements by photcode, or equiv photcode, if specified */ - if (NphotcodesKeep > 0) { - found = FALSE; - for (kp = 0; (kp < NphotcodesKeep) && !found; kp++) { - if (photcodesKeep[kp][0].code == catalog[i].measure[m].photcode) found = TRUE; - if (photcodesKeep[kp][0].code == GetPhotcodeEquivCodebyCode(catalog[i].measure[m].photcode)) found = TRUE; - } - if (!found) continue; - } - if (NphotcodesSkip > 0) { - found = FALSE; - for (kp = 0; (kp < NphotcodesSkip) && !found; kp++) { - if (photcodesSkip[kp][0].code == catalog[i].measure[m].photcode) found = TRUE; - if (photcodesSkip[kp][0].code == GetPhotcodeEquivCodebyCode(catalog[i].measure[m].photcode)) found = TRUE; - } - if (found) continue; - } + catalog[i].measure[m].dbFlags |= ID_MEAS_USED_OBJ; R[N] = getMeanR (&catalog[i].measure[m], &catalog[i].average[j], &catalog[i].secfilt[j*Nsecfilt]); @@ -179,8 +145,8 @@ // too few measurements for average position (require 2 values) - if (N < 2) { + if (N < SRC_MEAS_TOOFEW) { // XXX need to define PHOTOM and ASTROM object flags - // catalog[i].average[j].code |= ID_STAR_FEW; - continue; + catalog[i].average[j].flags |= ID_STAR_FEW; + if (N < 2) continue; } @@ -276,4 +242,11 @@ } + //make sure that the fit succeeded + assert(finite(fit.Ro) && finite(fit.Do) && + finite(fit.dRo) && finite(fit.dDo) && + finite(fit.uR) && finite(fit.uD) && + finite(fit.duR) && finite(fit.duD) && + finite(fit.p) && finite(fit.dp)); + // the measure fields must be updated before the average fields m = catalog[i].average[j].measureOffset; Index: branches/eam_branches/20090522/Ohana/src/relastro/src/args.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/args.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/args.c (revision 24557) @@ -40,4 +40,13 @@ FIT_TARGET = TARGET_MOSAICS; } + + FlagOutlier = FALSE; + if ((N = get_argument (argc, argv, "-clip"))) { + remove_argument (N, &argc, argv); + CLIP_THRESH = atof (argv[N]); + remove_argument (N, &argc, argv); + FlagOutlier = TRUE; + } + if (FIT_TARGET == TARGET_NONE) usage(); Index: branches/eam_branches/20090522/Ohana/src/relastro/src/bcatalog.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/bcatalog.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/bcatalog.c (revision 24557) @@ -2,5 +2,5 @@ int bcatalog (Catalog *subcatalog, Catalog *catalog) { - + int i, j, k, offset, found; int NAVERAGE, NMEASURE, Naverage, Nmeasure, Nm, Nsecfilt; @@ -23,6 +23,8 @@ /* exclude stars not in range or with too few measurements */ for (i = 0; i < catalog[0].Naverage; i++) { - if (catalog[0].average[i].Nmeasure < 2) continue; - + if (catalog[0].average[i].Nmeasure <= SRC_MEAS_TOOFEW) { + continue; + } + /* start with all stars good */ subcatalog[0].average[Naverage] = catalog[0].average[i]; @@ -44,53 +46,17 @@ offset = catalog[0].average[i].measureOffset + j; - - /* select measurements by photcode, or equiv photcode, if specified */ - if (NphotcodesKeep > 0) { - found = FALSE; - for (k = 0; (k < NphotcodesKeep) && !found; k++) { - if (photcodesKeep[k][0].code == catalog[0].measure[offset].photcode) found = TRUE; - if (photcodesKeep[k][0].code == GetPhotcodeEquivCodebyCode(catalog[0].measure[offset].photcode)) found = TRUE; - } - if (!found) continue; - } - if (NphotcodesSkip > 0) { - found = FALSE; - for (k = 0; (k < NphotcodesSkip) && !found; k++) { - if (photcodesSkip[k][0].code == catalog[0].measure[offset].photcode) found = TRUE; - if (photcodesSkip[k][0].code == GetPhotcodeEquivCodebyCode(catalog[0].measure[offset].photcode)) found = TRUE; - } - if (found) continue; + + //filter objects based on user supplied criteria + if (!MeasFilterTest(&catalog[0].measure[offset])) { + catalog[0].measure[offset].dbFlags &= ~ID_MEAS_USED_CHIP; + continue; } - /* select measurements by time */ - if (TimeSelect) { - if (catalog[0].measure[offset].t < TSTART) continue; - if (catalog[0].measure[offset].t > TSTOP) continue; + //filter out outliers + if (FlagOutlier && (catalog[0].measure[offset].dbFlags & ID_MEAS_POOR_ASTROM)) { + catalog[0].measure[offset].dbFlags &= ~ID_MEAS_USED_CHIP; + continue; } - - /* select measurements by quality */ - // XXX FIX THIS!! - // if (DophotSelect && (catalog[0].measure[offset].dophot != DophotValue)) continue; - - /* select measurements by quality */ - if (PhotFlagSelect) { - if (PhotFlagBad) { - mask = PhotFlagBad; - } else { - code = GetPhotcodebyCode (catalog[0].measure[offset].photcode); - mask = code[0].astromBadMask; - } - if (mask & catalog[0].measure[offset].photFlags) continue; - } - - /* select measurements by measurement error */ - if ((SIGMA_LIM > 0) && (catalog[0].measure[offset].dM > SIGMA_LIM)) continue; - - /* select measurements by mag limit */ - if (ImagSelect) { - mag = PhotInst (&catalog[0].measure[offset]); - if (mag < ImagMin) continue; - if (mag > ImagMax) continue; - } + catalog[0].measure[offset].dbFlags |= ID_MEAS_USED_CHIP; // re-assess on each run of relastro if a measurement should be used @@ -110,13 +76,14 @@ subcatalog[0].measure[Nmeasure] = catalog[0].measure[offset]; subcatalog[0].measure[Nmeasure].averef = Naverage; - if (RESET) { - subcatalog[0].measure[Nmeasure].dbFlags &= ~ID_MEAS_POOR_ASTROM; - subcatalog[0].measure[Nmeasure].dbFlags &= ~ID_MEAS_AREA; + if (RESET) { + subcatalog[0].measure[Nmeasure].dbFlags &= ~ID_MEAS_POOR_ASTROM; + subcatalog[0].measure[Nmeasure].dbFlags &= ~ID_MEAS_AREA; } + Nmeasure ++; Nm ++; if (Nmeasure == NMEASURE) { - NMEASURE += 1000; - REALLOCATE (subcatalog[0].measure, Measure, NMEASURE); + NMEASURE += 1000; + REALLOCATE (subcatalog[0].measure, Measure, NMEASURE); } } @@ -139,6 +106,6 @@ if (VERBOSE) { - fprintf (stderr, "%d: using %d stars (%d measures) for catalog\n", i, - subcatalog[0].Naverage, subcatalog[0].Nmeasure); + fprintf (stderr, "%d: using %d stars (%d measures) for catalog\n", i, + subcatalog[0].Naverage, subcatalog[0].Nmeasure); } return (TRUE); Index: branches/eam_branches/20090522/Ohana/src/relastro/src/initialize.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/initialize.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/initialize.c (revision 24557) @@ -25,6 +25,6 @@ fprintf (stderr, "codename: %s\n", codename); if ((photcodesKeep[NphotcodesKeep] = GetPhotcodebyName (codename)) == NULL) { - fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); - exit (1); + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); } NphotcodesKeep ++; @@ -46,6 +46,6 @@ fprintf (stderr, "codename: %s\n", codename); if ((photcodesSkip[NphotcodesSkip] = GetPhotcodebyName (codename)) == NULL) { - fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); - exit (1); + fprintf (stderr, "ERROR: photcode %s not found in photcode table\n", codename); + exit (1); } NphotcodesSkip ++; Index: branches/eam_branches/20090522/Ohana/src/relastro/src/load_catalogs.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/load_catalogs.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/load_catalogs.c (revision 24557) @@ -29,4 +29,9 @@ } if (VERBOSE && !pcatalog[0].Naves_disk) fprintf (stderr, "no data in %s, skipping\n", pcatalog[0].filename); + + //outlier rejection + if (FlagOutlier) { + FlagOutliers(&tcatalog); + } // select only the brighter stars Index: branches/eam_branches/20090522/Ohana/src/relastro/src/relastroVisual.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/relastroVisual.c (revision 24557) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/relastroVisual.c (revision 24557) @@ -0,0 +1,512 @@ +/** Diagnostic plots for relastro + * @author Chris Beaumont, IfA + */ + +#include "kapa.h" +#include "relastro.h" + +#define TESTING + +#define KAPAX 700 +#define KAPAY 700 + +static int kapa = -1; +static int kapa2 = -1; +static int kapa3 = -1; + +static int isVisual = FALSE; +static int plotRawRef = FALSE; +static int plotScatter = FALSE; +static int plotResid = FALSE; +static int plotVector = FALSE; +static int plotOutliers = TRUE; + +/** Spawn a kapa window */ +static int initWindow(int *kapid) { + if (*kapid == -1) { + *kapid = KapaOpenNamedSocket("kapa", "relastro_plots"); + if (*kapid == -1) { + fprintf(stderr, "Failure to open kapa.\n"); + isVisual = 0; + return 0; + } + // KapaResize (*kapid, KAPAX, KAPAY); + } + return 1; +} + +/** Ask the user how to proceed */ +static int askUser(int *flag) { + char key[10]; + fprintf (stdout, "[c]ontinue? [s]kip the rest of these plots? [a]bort all visual plots?"); + if (!fgets(key, 8, stdin)) { + fprintf(stderr, "warning: Unable to read option"); + } + if (key[0] == 's') { + *flag = 0; + } + if (key[0] == 'a') { + isVisual = 0; + } + return 1; +} + +/** Size graphdata to encompass all points */ +static int scaleGraphdata(float x[], float y[], Graphdata *graphdata, int n) { + float xlo = FLT_MAX, xhi = -FLT_MAX, ylo = FLT_MAX, yhi = -FLT_MAX; + int i; + int goodData = 0; + for(i = 0; i < n; i++) { + goodData++; + if(!finite(x[i]) || !finite(y[i])) continue; + if(x[i] < xlo) xlo = x[i]; + if(x[i] > xhi) xhi = x[i]; + if(y[i] < ylo) ylo = y[i]; + if(y[i] > yhi) yhi = y[i]; + } + if (goodData < 2) return 0; + graphdata->xmin = xlo; + graphdata->ymin = ylo; + graphdata->xmax = xhi; + graphdata->ymax = yhi; + return 1; +} + +static int residPlot(float x[], float y[], + float xVec[], float yVec[], + int npts, int *kapaID) { + if (!isVisual || !plotResid) return TRUE; + Graphdata graphdata; + KapaSection section; + + if (!initWindow(kapaID)) return FALSE; + + KapaInitGraph(&graphdata); + KapaClearPlots(*kapaID); + KapaSetFont(*kapaID, "helvetica", 14); + + section.name = "0"; + section.x = 0.0; section.y = 0.0; + section.dx = .45, section.dy = .45; + graphdata.ptype = 7; + graphdata.style = 2; + + KapaSetSection(*kapaID, §ion); + if(!scaleGraphdata(x, xVec, &graphdata, npts)) return 0; + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, "x", KAPA_LABEL_XM); + KapaSendLabel(*kapaID, "dx", KAPA_LABEL_YM); + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, x, "x"); + KapaPlotVector(*kapaID, npts, xVec, "y"); + + section.x = .5; section.y = 0; section.name="1"; + KapaSetSection(*kapaID, §ion); + if(!scaleGraphdata(x, yVec, &graphdata, npts)) return 0; + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, "x", KAPA_LABEL_XM); + KapaSendLabel(*kapaID, "dy", KAPA_LABEL_YP); + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, x, "x"); + KapaPlotVector(*kapaID, npts, yVec, "y"); + + section.x = .0; section.y = .5; section.name="2"; + KapaSetSection(*kapaID, §ion); + if(!scaleGraphdata(y, xVec, &graphdata, npts)) return 0;; + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, "y", KAPA_LABEL_XM); + KapaSendLabel(*kapaID, "dx", KAPA_LABEL_YM); + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, y, "x"); + KapaPlotVector(*kapaID, npts, xVec, "y"); + + section.x = .5; section.y = .5; section.name="3"; + KapaSetSection(*kapaID, §ion); + if(!scaleGraphdata(y, yVec, &graphdata, npts)) return 0; + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, "y", KAPA_LABEL_XM); + KapaSendLabel(*kapaID, "dy", KAPA_LABEL_YP); + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, y, "x"); + KapaPlotVector(*kapaID, npts, yVec, "y"); + + return 1; +} + + +/**Plot a vector field*/ +static int plotVectorField(float x[], float y[], + float xVec[], float yVec[], + int npts, int *kapaID, double maxVecLength) { + + if(!plotVector) return 1; + + Graphdata graphdata; + float singleX[2], singleY[2]; + float vecScaleFactor; + float graphSize; + int i; + char plotTitle[50]; + sprintf(plotTitle, "Maximum Vector Size = %5.1e", maxVecLength); + + if (!initWindow(kapaID)) return 0; + + KapaInitGraph(&graphdata); + KapaClearPlots(*kapaID); + if(!scaleGraphdata(x, y, &graphdata, npts)) return 0; + + graphSize = graphdata.xmax - graphdata.xmin; + if ((graphdata.ymax - graphdata.ymin) > graphSize) { + graphSize = graphdata.ymax - graphdata.ymin; + } + + vecScaleFactor = graphSize * 0.02 / (float)maxVecLength; +#ifdef TESTING + fprintf(stderr, "GraphSize: %e\n maxVecLength: %e\n vecScaleFactor: %e\n", + graphSize, maxVecLength, vecScaleFactor); +#endif + + + KapaSetFont (*kapaID, "helvetica", 14); + KapaSetLimits(*kapaID, &graphdata); + KapaBox(*kapaID, &graphdata); + KapaSendLabel(*kapaID, plotTitle, KAPA_LABEL_XP); + + graphdata.ptype = 7; + graphdata.style = 2; + KapaPrepPlot(*kapaID, npts, &graphdata); + KapaPlotVector(*kapaID, npts, x, "x"); + KapaPlotVector(*kapaID, npts, y, "y"); + + //plot each vector individually + graphdata.ptype = 0; + graphdata.style = 0; + graphdata.color = KapaColorByName("blue"); + for(i = 0; i < npts; i++) { + singleX[0] = x[i]; + singleY[0] = y[i]; + singleX[1] = x[i] + xVec[i] * vecScaleFactor; + singleY[1] = y[i] + yVec[i] * vecScaleFactor; + KapaPrepPlot(*kapaID, 2, &graphdata); + KapaPlotVector(*kapaID, 2, singleX, "x"); + KapaPlotVector(*kapaID, 2, singleY, "y"); + } + return 1; +} + +int relastroVisualPlotScatter(double values[], double thresh, int npts) { + float *x, *data; + int i; + float xline[2], yline[2]; + if (!isVisual || !plotScatter) return 1; + if (!initWindow(&kapa2)) return 0; + + ALLOCATE(x, float, npts); + ALLOCATE(data, float, npts); + + for(i = 0; i < npts; i++) { + x[i] = i; + data[i] = (float) values[i]; + } + + Graphdata graphdata; + KapaSection section; + section.x = 0; section.y = 0; section.dx = 1; section.dy = 1; + section.name = "junk"; + + KapaInitGraph(&graphdata); + KapaClearPlots(kapa2); + KapaSetSection(kapa2, §ion); + + graphdata.ptype = 0; + graphdata.style = 0; + graphdata.xmin = 0; + graphdata.xmax = npts; + graphdata.ymin = 0; + graphdata.ymax = data[npts-1]; + + KapaSetFont(kapa2, "helvetica", 14); + KapaSetLimits(kapa2, &graphdata); + KapaBox(kapa2, &graphdata); + KapaSendLabel( kapa2, "Object", KAPA_LABEL_XM); + KapaSendLabel( kapa2, "Offset(pixels)", KAPA_LABEL_YM); + KapaSendLabel( kapa2, "Astrometric Offset with fit cutoff", + KAPA_LABEL_XP); + KapaPrepPlot(kapa2, npts, &graphdata); + KapaPlotVector(kapa2, npts, x, "x"); + KapaPlotVector(kapa2, npts, data, "y"); + + graphdata.color = KapaColorByName("red"); + KapaPrepPlot(kapa2, 2, &graphdata); + yline[0] = (float) thresh; + yline[1] = (float) thresh; + xline[0] = graphdata.xmin; + xline[1] = graphdata.xmax; + KapaPlotVector(kapa2, 2, xline, "x"); + KapaPlotVector(kapa2, 2, yline, "y"); + + askUser(&plotScatter); + return 1; +} + + + +/** plot raw vs ref (L, M). Only those whose distance is < drMax are used in fit*/ +int relastroVisualPlotRawRef(StarData *raw, StarData *ref, double dRmax, int numObj) { + + if( !isVisual || !plotRawRef) return 1; + if( !initWindow(&kapa)) return 0; + + float *rawX, *rawY, *refX, *refY; + float *rawXfit, *rawYfit, *refXfit, *refYfit; + float *magRaw, *magRef, *magRawfit, *magReffit; + float *xVec, *yVec; + int numFit = 0, numNoFit = 0; + double dL, dM, dR; + + ALLOCATE(rawX, float, numObj); + ALLOCATE(rawY, float, numObj); + ALLOCATE(refX, float, numObj); + ALLOCATE(refY, float, numObj); + ALLOCATE(rawXfit, float, numObj); + ALLOCATE(rawYfit, float, numObj); + ALLOCATE(refXfit, float, numObj); + ALLOCATE(refYfit, float, numObj); + ALLOCATE(magRaw, float, numObj); + ALLOCATE(magRef, float, numObj); + ALLOCATE(magRawfit, float, numObj); + ALLOCATE(magReffit, float, numObj); + + int i; + for(i = 0; i < numObj; i++) { + if (raw[i].mask) continue; + + dL = raw[i].L - ref[i].L; + dM = raw[i].M - ref[i].M; + dR = hypot (dL, dM); + if (dR > dRmax) { + rawX[numNoFit] = raw[i].X; + rawY[numNoFit] = raw[i].Y; + refX[numNoFit] = ref[i].X; + refY[numNoFit] = ref[i].Y; + magRaw[numNoFit] = raw[i].Mag; + magRef[numNoFit] = ref[i].Mag; + numNoFit++; + } else { + rawXfit[numFit] = raw[i].X; + rawYfit[numFit] = raw[i].Y; + refXfit[numFit] = ref[i].X; + refYfit[numFit] = ref[i].Y; + magRaw[numFit] = raw[i].Mag; + magRef[numFit] = ref[i].Mag; + numFit++; + } + } + + if (numFit == 0) return 0; + + Graphdata graphdata; + + KapaInitGraph(&graphdata); + KapaClearPlots(kapa); + + graphdata.ptype = 7; + graphdata.style = 2; + + if(!scaleGraphdata(rawXfit, rawYfit, &graphdata, numFit)) { + fprintf(stderr, "Not enough finite points for plotting"); + return 0; + } + + KapaSetFont(kapa, "helvetica", 14); + KapaSetLimits(kapa, &graphdata); + KapaBox(kapa, &graphdata); + KapaSendLabel( kapa, "X", KAPA_LABEL_XM); + KapaSendLabel( kapa, "Y", KAPA_LABEL_YM); + KapaSendLabel( kapa, "orange, red, green, blue: (raw, ref), (nofit, fit)", + KAPA_LABEL_XP); + + graphdata.color = KapaColorByName("orange"); + graphdata.size = 1; + KapaPrepPlot(kapa, numNoFit, &graphdata); + KapaPlotVector(kapa, numNoFit, rawX, "x"); + KapaPlotVector(kapa, numNoFit, rawY, "y"); + + graphdata.color = KapaColorByName("red"); + graphdata.size = 2; + KapaPrepPlot(kapa, numNoFit, &graphdata); + KapaPlotVector(kapa, numNoFit, refX, "x"); + KapaPlotVector(kapa, numNoFit, refY, "y"); + + graphdata.color = KapaColorByName("green"); + graphdata.size = 1; + KapaPrepPlot(kapa, numFit, &graphdata); + KapaPlotVector(kapa, numFit, rawXfit, "x"); + KapaPlotVector(kapa, numFit, rawYfit, "y"); + + graphdata.color = KapaColorByName("blue"); + graphdata.size = 2; + KapaPrepPlot(kapa, numFit, &graphdata); + KapaPlotVector(kapa, numFit, refXfit, "x"); + KapaPlotVector(kapa, numFit, refYfit, "y"); + + ALLOCATE(xVec, float, numFit); + ALLOCATE(yVec, float, numFit); + + //plot the fitted objects as vectors + for(i = 0; i < numFit; i++) { + xVec[i] = rawXfit[i] - refXfit[i]; + yVec[i] = rawYfit[i] - refYfit[i]; + } + + plotVectorField(rawXfit, rawYfit, xVec, yVec, numFit, &kapa3, dRmax); + if(!residPlot(rawXfit, rawYfit, xVec, yVec, numFit, &kapa2)) { + fprintf(stderr, "Unable to plot residuals"); + return 0; + } + + FREE(xVec); + FREE(yVec); + + askUser(&plotRawRef); + + FREE(rawX); + FREE(rawY); + FREE(refX); + FREE(refY); + FREE(rawXfit); + FREE(rawYfit); + FREE(refXfit); + FREE(refYfit); + FREE(magRaw); + FREE(magRef); + FREE(magRawfit); + FREE(magReffit); + + return 1; +} + +int relastroVisualPlotOutliers(Catalog *catalog, int offset, int Nmeasure, + StatType statsR, StatType statsD, double thresh) { + + float *Din, *Rin, *Dout, *Rout; + double xmin, xmax, ymin, ymax, range; + float xCirc[100], yCirc[100]; + int m, i; + int Nin, Nout; + Measure meas; + Graphdata graphdata; + KapaSection section; + + if (!isVisual || !plotOutliers) return 1; + if (!initWindow(&kapa)) return 0; + + // populate vectors + ALLOCATE(Din, float, Nmeasure); + ALLOCATE(Rin, float, Nmeasure); + ALLOCATE(Dout, float, Nmeasure); + ALLOCATE(Rout, float, Nmeasure); + + //create the threshhold ellipse + for(i = 0; i < 100; i++) { + xCirc[i] = statsR.median + thresh * statsR.sigma * cos(2 * 3.14 / 99. * i); + yCirc[i] = statsD.median + thresh * statsD.sigma * sin(2 * 3.14 / 99. * i); + } + + m = offset; + Nin = Nout = 0; + xmin = +FLT_MAX; + xmax = -FLT_MAX; + ymax = -FLT_MAX; + ymin = +FLT_MAX; + for(i = 0; i < Nmeasure; i++, m++) { + meas = catalog[0].measure[m]; + if (!MeasFilterTest(&meas)) continue; + xmin = MIN(xmin, meas.dR); + xmax = MAX(xmax, meas.dR); + ymin = MIN(ymin, meas.dD); + ymax = MAX(ymax, meas.dD); + + if (meas.dbFlags & ID_MEAS_POOR_ASTROM) { + Rout[Nout] = (float)(meas.dR); + Dout[Nout] = (float)(meas.dD); + fprintf(stderr, "r: %f\td: %f\t outlier: 1\n", Rout[Nout], Dout[Nout]); + Nout++; + } else { + Rin[Nin] = (float)(meas.dR); + Din[Nin] = (float)(meas.dD); + fprintf(stderr, "r: %f\td: %f\t outlier: 0\n", Rin[Nin], Din[Nin]); + Nin++; + } + } + + range = (xmax - xmin); + xmin -= .1 * range; + xmax += .1 * range; + range = (ymax - ymin); + ymax += .1 * range; + ymin -= .1 * range; + + //temporary fix + xmin = -1; xmax = 1; ymin = -1; ymax = 1; + + + //initialize graph info + section.x = 0; section.y = 0; section.dx = 1; section.dy = 1; + section.name = "junk"; + + KapaInitGraph(&graphdata); + KapaClearPlots(kapa); + KapaSetFont(kapa, "helvetica", 14); + + graphdata.ptype = 7; + graphdata.style = 2; + graphdata.size = 3; + graphdata.xmin = xmin; + graphdata.xmax = xmax; + graphdata.ymin = ymin; + graphdata.ymax = ymax; + + KapaSetSection(kapa, §ion); + KapaSetLimits(kapa, &graphdata); + KapaBox(kapa, &graphdata); + + KapaSendLabel( kapa, "RA (arcsec)", KAPA_LABEL_XM); + KapaSendLabel( kapa, "Dec (arcsec)", KAPA_LABEL_YM); + KapaSendLabel( kapa, "Points flagged as outliers (red)", + KAPA_LABEL_XP); + + graphdata.color = KapaColorByName("green"); + KapaPrepPlot(kapa, Nin, &graphdata); + KapaPlotVector(kapa, Nin, Rin, "x"); + KapaPlotVector(kapa, Nin, Din, "y"); + + graphdata.color = KapaColorByName("red"); + KapaPrepPlot(kapa, Nout, &graphdata); + KapaPlotVector(kapa, Nout, Rout, "x"); + KapaPlotVector(kapa, Nout, Dout, "y"); + + graphdata.color = KapaColorByName("black"); + graphdata.ptype = 0; + graphdata.style = 0; + KapaPrepPlot(kapa, 100, &graphdata); + KapaPlotVector(kapa, 100, xCirc, "x"); + KapaPlotVector(kapa, 100, yCirc, "y"); + + + + askUser(&plotOutliers); + + FREE(Rout); + FREE(Dout); + FREE(Rin); + FREE(Din); + + return TRUE; +} + + Index: branches/eam_branches/20090522/Ohana/src/relastro/src/relastroVisual.h =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/relastroVisual.h (revision 24557) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/relastroVisual.h (revision 24557) @@ -0,0 +1,9 @@ +#ifndef RELASTRO_VISUAL_H +#define RELASTROPVISUAL_H + +int relastroVisualPlotRawRef(StarData *raw, StarData *ref, double dRmax, int numObj); +int relastroVisualPlotScatter(double values[], double thresh, int npts); +int relastroVisualPlotOutliers(Catalog *catalog, int offset, int Nmeasure, + StatType statsR, StatType statsD, double thresh); + +#endif Index: branches/eam_branches/20090522/Ohana/src/relastro/src/relastro_objects.c =================================================================== --- branches/eam_branches/20090522/Ohana/src/relastro/src/relastro_objects.c (revision 24529) +++ branches/eam_branches/20090522/Ohana/src/relastro/src/relastro_objects.c (revision 24557) @@ -8,4 +8,5 @@ SkyList *skylist = NULL; Catalog catalog; + // load the current sky table (layout of all SkyRegions) @@ -37,4 +38,8 @@ } + if (FlagOutlier) { + FlagOutliers(&catalog); + } + // XXX consider what gets reset (only ASTROM flags) if (RESET) { Index: branches/eam_branches/20090522/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- branches/eam_branches/20090522/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 24529) +++ branches/eam_branches/20090522/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 24557) @@ -370,4 +370,6 @@ my $proj_id = $ref->{proj_id}; $stmt->finish(); + ( carp "proj_id for $dbname not found in $admindb" and return undef ) unless $proj_id; + my $source_id = ($proj_id << 3) | $table; @@ -1268,18 +1270,18 @@ # check if file actually exists if ($self->file_exists( $outname )) { - # double check that the file is not zero-length (eg: dvoImageExtract crashed) + # double check that the file is not zero-length (eg: dvoImageExtract crashed) $outnameResolved = $self->file_resolve( $outname, 1 ) or return undef; # Resolved filename, for Nebulous - my @stats = stat($outnameResolved); - if ($stats[7]) { - return 1; - } + my @stats = stat($outnameResolved); + if ($stats[7]) { + return 1; + } } unless (defined $outnameResolved) { - $outnameResolved = $self->file_create( $outname ) or return undef; # Resolved filename, for Nebulous + $outnameResolved = $self->file_create( $outname ) or return undef; # Resolved filename, for Nebulous } my $command = "$dvoImageExtract -D CATDIR $tess_dir $skycell_id -o $outnameResolved"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => $verbose); ( carp "Unable to perform dvoImageExtract for $tess_id $skycell_id\n" and return undef ) unless ($success and $self->file_exists( $outname )); Index: branches/eam_branches/20090522/dbconfig/changes.txt =================================================================== --- branches/eam_branches/20090522/dbconfig/changes.txt (revision 24529) +++ branches/eam_branches/20090522/dbconfig/changes.txt (revision 24557) @@ -1126,2 +1126,72 @@ -- Version 1.1.53 + +ALTER TABLE warpRun ADD COLUMN reduction VARCHAR(64) AFTER tess_id; + +-- Tables to support publishing of detections to a Science Client +-- Clients to which we send stuff +CREATE TABLE publishClient ( + client_id BIGINT AUTO_INCREMENT, -- unique identifier + product VARCHAR(64), -- product name + stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + comment VARCHAR(255), -- for human memory + PRIMARY KEY(client_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; +-- Publishing a set of data (e.g., a specific diffRun) +CREATE TABLE publishRun ( + pub_id BIGINT AUTO_INCREMENT, -- unique identifier + client_id BIGINT NOT NULL, -- link to publishClient + stage_id BIGINT NOT NULL, -- link to various stage tables + label VARCHAR(64), -- label for run + state VARCHAR(64), -- state of run (new, full, etc.) + PRIMARY KEY(pub_id), + KEY(client_id), + KEY(stage_id), + KEY(label), + KEY(state), + FOREIGN KEY(client_id) REFERENCES publishClient(client_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; +-- Publishing a file within a set +CREATE TABLE publishDone ( + pub_id BIGINT AUTO_INCREMENT, -- link to publishRun + path_base VARCHAR(255), -- base path of output + fault SMALLINT NOT NULL DEFAULT 0, -- Fault code + PRIMARY KEY(pub_id), + KEY(fault), + FOREIGN KEY(pub_id) REFERENCES publishRun(pub_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Version 1.1.54 + +-- changed magicked from a flag to a BIGINT +-- The value is the magic_ds_id of the magicDSRun that destreaked the file/run + +ALTER TABLE magicDSFile ADD COLUMN data_state VARCHAR(64); +ALTER TABLE magicDSFile ADD KEY(data_state); +UPDATE magicDSFile SET data_state = 'full' WHERE data_state IS NULL; + +ALTER TABLE rawImfile CHANGE COLUMN magicked magicked BIGINT; +ALTER TABLE rawExp CHANGE COLUMN magicked magicked BIGINT; + +ALTER TABLE chipProcessedImfile CHANGE COLUMN magicked magicked BIGINT; +ALTER TABLE chipRun CHANGE COLUMN magicked magicked BIGINT; + +-- so far only chipRuns have been magicked in product +UPDATE chipProcessedImfile JOIN magicDSRun ON stage = 'chip' AND chip_id = stage_id + SET chipProcessedImfile.magicked = magic_ds_id + WHERE magicDSRun.re_place + AND chipProcessedImfile.magicked > 0; + +UPDATE chipRun JOIN magicDSRun ON stage = 'chip' AND chip_id = stage_id + SET chipRun.magicked = magic_ds_id + WHERE magicDSRun.re_place + AND chipRun.magicked > 0; + +ALTER TABLE warpSkyfile CHANGE COLUMN magicked magicked BIGINT; +ALTER TABLE warpRun CHANGE COLUMN magicked magicked BIGINT; + +ALTER TABLE diffSkyfile CHANGE COLUMN magicked magicked BIGINT; +ALTER TABLE diffRun CHANGE COLUMN magicked magicked BIGINT; + + Index: branches/eam_branches/20090522/dbconfig/chip.md =================================================================== --- branches/eam_branches/20090522/dbconfig/chip.md (revision 24529) +++ branches/eam_branches/20090522/dbconfig/chip.md (revision 24557) @@ -11,5 +11,5 @@ tess_id STR 64 end_stage STR 64 # Key - magicked BOOL f + magicked S64 0 END @@ -89,5 +89,5 @@ fault S16 0 # Key NOT NULL quality S16 0 - magicked BOOL f + magicked S64 0 END Index: branches/eam_branches/20090522/dbconfig/config.md =================================================================== --- branches/eam_branches/20090522/dbconfig/config.md (revision 24529) +++ branches/eam_branches/20090522/dbconfig/config.md (revision 24557) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.53 + pkg_version STR 1.1.54 END Index: branches/eam_branches/20090522/dbconfig/diff.md =================================================================== --- branches/eam_branches/20090522/dbconfig/diff.md (revision 24529) +++ branches/eam_branches/20090522/dbconfig/diff.md (revision 24557) @@ -12,5 +12,5 @@ bothways BOOL f exposure BOOL f - magicked BOOL f + magicked S64 0 END @@ -57,4 +57,4 @@ fault S16 0 # Key quality S16 0 - magicked BOOL f + magicked S64 0 END Index: branches/eam_branches/20090522/dbconfig/ipp.m4 =================================================================== --- branches/eam_branches/20090522/dbconfig/ipp.m4 (revision 24529) +++ branches/eam_branches/20090522/dbconfig/ipp.m4 (revision 24557) @@ -26,2 +26,3 @@ include(rc.md) include(receive.md) +include(publish.md) Index: branches/eam_branches/20090522/dbconfig/magic.md =================================================================== --- branches/eam_branches/20090522/dbconfig/magic.md (revision 24529) +++ branches/eam_branches/20090522/dbconfig/magic.md (revision 24557) @@ -64,3 +64,4 @@ recovery_path_base STR 255 fault S16 0 + data_state STR 64 # Key END Index: branches/eam_branches/20090522/dbconfig/publish.md =================================================================== --- branches/eam_branches/20090522/dbconfig/publish.md (revision 24557) +++ branches/eam_branches/20090522/dbconfig/publish.md (revision 24557) @@ -0,0 +1,23 @@ +# Tables for publishing data to a science client + +publishClient METADATA + client_id S64 0 # Primary Key AUTO_INCREMENT + product STR 64 + stage STR 64 + workdir STR 255 + comment STR 255 +END + +publishRun METADATA + pub_id S64 0 # Primary Key AUTO_INCREMENT + client_id S64 0 + stage_id S64 0 + label STR 64 + state STR 64 +END + +publishDone METADATA + pub_id S64 0 # Primary Key + path_base STR 255 + fault S16 0 +END Index: branches/eam_branches/20090522/dbconfig/raw.md =================================================================== --- branches/eam_branches/20090522/dbconfig/raw.md (revision 24529) +++ branches/eam_branches/20090522/dbconfig/raw.md (revision 24557) @@ -65,5 +65,5 @@ fault S16 0 # Key NOT NULL epoch UTC 0001-01-01T00:00:00Z - magicked BOOL f + magicked S64 0 END @@ -135,4 +135,4 @@ quality S16 0 epoch UTC 0001-01-01T00:00:00Z - magicked BOOL f + magicked S64 0 END Index: branches/eam_branches/20090522/dbconfig/warp.md =================================================================== --- branches/eam_branches/20090522/dbconfig/warp.md (revision 24529) +++ branches/eam_branches/20090522/dbconfig/warp.md (revision 24557) @@ -19,7 +19,8 @@ dvodb STR 255 tess_id STR 64 + reduction STR 64 end_stage STR 64 # Key registered TAI NULL - magicked BOOL f + magicked S64 0 END @@ -57,5 +58,5 @@ fault S16 0 # Key quality S16 0 - magicked BOOL f + magicked S64 0 END Index: branches/eam_branches/20090522/extsrc/gpcsw/Makefile.in =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/Makefile.in (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/Makefile.in (revision 24557) @@ -2,5 +2,5 @@ default: all -all: burntool +all: burntool tiltystreak install: all @@ -24,4 +24,12 @@ svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/libfhreg gpcsrc/fits/libfhreg.new svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/burntool gpcsrc/fits/burntool.new + svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/tiltystreak gpcsrc/fits/tiltystreak.new +# ensure the target directory exists + mkdir -p gpcsrc/analysis/libpscoords + mkdir -p gpcsrc/analysis/libpsf + mkdir -p gpcsrc/fits/libfh + mkdir -p gpcsrc/fits/libfhreg + mkdir -p gpcsrc/fits/burntool + mkdir -p gpcsrc/fits/tiltystreak # replace new files with those from the imported directories cp -rf gpcsrc/analysis/libpscoords.new/* gpcsrc/analysis/libpscoords @@ -30,4 +38,5 @@ cp -rf gpcsrc/fits/libfhreg.new/* gpcsrc/fits/libfhreg cp -rf gpcsrc/fits/burntool.new/* gpcsrc/fits/burntool + cp -rf gpcsrc/fits/tiltystreak.new/* gpcsrc/fits/tiltystreak # remove the temp directory rm -rf gpcsrc/analysis/libpscoords.new @@ -36,4 +45,5 @@ rm -rf gpcsrc/fits/libfhreg.new rm -rf gpcsrc/fits/burntool.new + rm -rf gpcsrc/fits/tiltystreak.new # modify burntool to avoid libpsf (which requires fortran) cp -f burntool.nopsf/Makefile gpcsrc/fits/burntool @@ -48,4 +58,5 @@ ln -sf ../Make.Common gpcsrc/fits/libfhreg/Make.Common ln -sf ../Make.Common gpcsrc/fits/burntool/Make.Common + ln -sf ../Make.Common gpcsrc/fits/tiltystreak/Make.Common burntool: @@ -54,2 +65,7 @@ cd gpcsrc && make -C analysis/libpscoords install cd gpcsrc && make -C fits/burntool install + +tiltystreak: + cd gpcsrc && make -C fits/libfh install + cd gpcsrc && make -C fits/libfhreg install + cd gpcsrc && make -C fits/tiltystreak install Index: branches/eam_branches/20090522/extsrc/gpcsw/burntool.nopsf/psfstats.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/burntool.nopsf/psfstats.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/burntool.nopsf/psfstats.c (revision 24557) @@ -10,10 +10,9 @@ #include "burntool.h" #include "math.h" -#include "psf/psf.h" /****************************************************************/ /* psf_stats(): Tell us about this PSF star */ -STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias, - double *fwhm, double *q) +STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias, + double *fwhm, double *q) { return(1); Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/man/libpscoords.1 =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/man/libpscoords.1 (revision 24557) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/man/libpscoords.1 (revision 24557) @@ -0,0 +1,136 @@ +.nf +NAME: + libpscoords - convert all the various Pan-STARRS coordinate systems + +API: + pscoords [args] < infile > outfile + +DESCRIPTION: + +There are five coordinate systems used by Pan-STARRS: + + * Sky coordinates of conventional RA, Dec, and PA; + * Tangent plane coordinates (meaning the angles at which rays enter the + optical system, relative to the optical system itself); + * Focal plane coordinates (meaning physical position of an image on the + focal plane); + * OTA coordinates (meaning the pixel position of an image on the silicon + of a given OTA); and + * Cell coordinates (meaning the pixel position of an image on a given + cell). + +libpscoords has routines to convert between these various coordinate +systems. + +Normal Application +------------------ + +To go from a position (a,d) relative to (a0,d0,pa) pointing to cell coords: +--------------------------------------------------------------------------- + psc_tproject(a, d, a0, d0, pa, density, pi/2-alt, vertical, &x, &y); + psc_psoptics2(x/sec2rad, y/sec2rad, dx, dy, dpa, pscale, d2, d3, &xfp, &yfp); + psc_fp_to_pixel(xfp, yfp, &ota_xid, &ota_yid, &xota, &yota); + psc_pixel_to_cell(xota, yota, &cell_xid, &cell_yid, &xcell, &ycell); + +(skipping the d2 distortion term one can also use) + psc_psoptics(x/sec2rad, y/sec2rad, dx, dy, dpa, pscale, d3, &xfp, &yfp); + +To go from cell coords to a position (a,d) relative to (a0,d0,pa) pointing: +--------------------------------------------------------------------------- + psc_cell_to_pixel(cell_xid,cell_yid, xcell,ycell, &xota,&yota); + psc_pixel_to_fp(ota_xid, ota_yid, xota, yota, &xfp, &yfp); + psc_invoptics2(xfp, yfp, dx, dy, dpa, pscale, d2, d3, &x, &y); + psc_tplonglat(x*sec2rad, y*sec2rad, a0, d0, pa, density, pi/2-alt, vertical, &a, &d); + +(skipping the d2 distortion term one can also use) + psc_invoptics(xfp, yfp, dx, dy, dpa, pscale, d3, &x, &y); + +See the prototypes for suggested parameter values. Note that for +Pan-STARRS the "vertical" argument is the ROT angle (in rad). + + +Prototypes +---------- + +/* Enable application of chip offsets? */ +int psc_do_chipoff(int doit); /* Apply chip offsets? (0/1, default=0) */ + +/* Convert a focal plane position to OTA otax,otay, Cell cellx,celly, Pixel */ +int psc_fp_to_pixel(double xfp_um, double yfp_um, /* FP position (um) */ + int *ota_xid, int *ota_yid, /* OTA ID (0:7) */ + double *xota_pix, double *yota_pix); /* OTA position (pix) */ + +/* Convert a pixel position in OTA ota_xid,ota_yid to FP */ +int psc_pixel_to_fp(int ota_xid, int ota_yid, /* OTA ID (0:7) */ + double xota_pix, double yota_pix, /* OTA position (pix) */ + double *xfp_um, double *yfp_um); /* FP position (um) */ + +/* Convert an OTA pixel position to cell ID and cell coords */ +int psc_pixel_to_cell(double xota_pix, double yota_pix, /* OTA position (pix) */ + int *cell_xid, int *cell_yid, /* Cell ID (0:7) or -1 */ + double *xcell_pix, double *ycell_pix);/* Cell position (pix) */ + +/* Convert a cell position and ID to OTA coords */ +int psc_cell_to_pixel(int cell_xid, int cell_yid, /* Cell ID (0:7) */ + double xcell_pix, double ycell_pix, /* Cell position (pix) */ + double *xota_pix, double *yota_pix); /* OTA position (pix) */ + +/* Convert tangent plane (x,y) at (a0,d0,pa) to RA,Dec (a,d) [radians] */ +/* y is rotated CCW by pa from N; NOTE: x is west when PA = 0 (pos parity) */ +/* Set density=0 to apply no correction for differential refraction, otherwise P/T in STP */ +int psc_tplonglat(double x_rad, double y_rad, double a0_rad, double d0_rad, double pa_rad, + double density/*0.71 for HK*/, double zenith_rad/*pi/2-ALT*/, + double vertical_rad/*ROT*/, double *a_rad, double *d_rad); + +/* Project RA,Dec (a,d) to the tangent plane (x,y) at (a0,d0,pa) [radians] */ +/* y is rotated CCW by pa from N; NOTE: x is west when PA = 0 (pos parity) */ +/* Set density=0 to apply no correction for differential refraction, otherwise P/T in STP */ +int psc_tproject(double a_rad, double d_rad, double a0_rad, double d0_rad, double pa_rad, + double density/*0.71 for HK*/, double zenith_rad/*pi/2-ALT*/, + double vertical_rad/*ROT*/, double *x_rad, double *y_rad); + +/* Evaluate an inverse optical model: microns -> arcsec notionally */ +/* Offset, scale with *small* distortion, rotate */ +int psc_invoptics(double xfp_um, double yfp_um, double dx_sec, double dy_sec, double dpa_rad, + double pscale/*38.86um/sec*/, double d3/*1.49e-10sec^-2*/, + double *x_sec, double *y_sec); + +/* Evaluate an optical model: arcsec -> microns notionally */ +/* Offset, scale with distortion, rotate */ +int psc_psoptics(double x_sec, double y_sec, double dx_sec, double dy_sec, double dpa_rad, + double pscale/*38.86um/sec*/, double d3/*1.49e-10sec^-2*/, + double *xfp_um, double *yfp_um); + +/* Evaluate an inverse optical model: microns -> arcsec notionally */ +/* Offset, scale with *small* distortion, rotate */ +int psc_invoptics2(double xfp_um, double yfp_um, double dx_sec, double dy_sec, double dpa_rad, + double pscale/*38.793um/sec*/, + double d2/*9.8e-7sec^-2*/, double d3/*3.1e-11sec^-2*/, + double *x_sec, double *y_sec); + +/* Evaluate an optical model: arcsec -> microns notionally */ +/* Offset, scale with distortion, rotate */ +int psc_psoptics2(double x_sec, double y_sec, double dx_sec, double dy_sec, double dpa_rad, + double pscale/*38.793um/sec*/, + double d2/*9.8e-7sec^-2*/, double d3/*3.1e-11sec^-2*/, + double *xfp_um, double *yfp_um); + +/* Enable application of chip offsets (um, mrad)? */ +int psc_load_otaoff(const char *fname); + + + + + + +BUGS: + 090516: Conceivably a tiny bug in Sky<->TP projections very near + the pole, equally conceivably a bug in libm. + +SEE ALSO: + pscoords + +AUTHORS: + John Tonry + +VERSION: Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.c (revision 24557) @@ -24,9 +24,47 @@ rcsid[] = "$Id: pscoords.c,v 1.2 2008/09/06 03:26:48 jt Exp jt $"; +#define MRAD2 (0.001) /* Convert from mrad to rad, used for PSC_OFFROT_T */ + /* Overall focal plane offset and rotation (um, rad) */ -const PSC_OFFROT_T psc_psc_fpoff={0.,0.,0.}; +PSC_OFFROT_T psc_psc_fpoff={0.,0.,0.}; /* Offsets and rotations of each OTA (um, millirad) */ -const PSC_OFFROT_T psc_otaoff[PSC_NX*PSC_NY]={ /* Offsets and rotations derived o4885 090228 */ +PSC_OFFROT_T psc_otaoff[PSC_NX*PSC_NY]={ /* Offsets and rotations derived o4885 090516 */ + { 0.0, 0.0, 0.00}, { -6.8, -32.4, -1.30}, /* OTA00 , OTA10 */ + { 15.7,-130.7, 0.07}, { -9.4, -43.5, -1.34}, /* OTA20 , OTA30 */ + { 17.8, 71.7, -0.48}, { 7.2, 41.3, -0.82}, /* OTA40 , OTA50 */ + { 22.1, -48.3, -1.74}, { 0.0, 0.0, 0.00}, /* OTA60 , OTA70 */ + { -23.0, -12.9, 0.68}, { -52.0, -52.2, 0.88}, /* OTA01 , OTA11 */ + { 28.8, -10.9, -0.73}, { -22.9, 18.1, -0.89}, /* OTA21 , OTA31 */ + { -13.8, 45.9, 0.90}, { -12.2, 49.7, -0.32}, /* OTA41 , OTA51 */ + { -31.9, -9.8, -1.60}, { 1.9, 8.0, 0.53}, /* OTA61 , OTA71 */ + { -27.2, 42.6, 2.73}, { 6.5, -52.6, 0.48}, /* OTA02 , OTA12 */ + { 10.1, -37.9, -1.53}, { 3.1,-139.0, 1.75}, /* OTA22 , OTA32 */ + { -21.4, 25.6, -1.12}, { -13.8, 0.9, 0.02}, /* OTA42 , OTA52 */ + { -1.8, -16.1, -2.06}, { 76.0, 31.7, 1.70}, /* OTA62 , OTA72 */ + { -15.7, 24.1, 0.14}, {-104.2, -76.8, 1.97}, /* OTA03 , OTA13 */ + { 75.4, 31.8, 1.04}, { -18.7, 14.6, 0.51}, /* OTA23 , OTA33 */ + { -30.6, 53.4, -0.78}, { 12.6, 24.4, 0.43}, /* OTA43 , OTA53 */ + { 8.0, -34.9, 1.44}, { 6.4, -73.2, -0.93}, /* OTA63 , OTA73 */ + { -31.2, 11.2, 0.23}, { -10.3, -2.1, 0.14}, /* OTA04 , OTA14 */ + { 33.2, 207.9, -0.40}, { -75.7,-105.9, 0.93}, /* OTA24 , OTA34 */ + { 8.1, -15.9, -0.03}, { -16.1, -13.3, -0.06}, /* OTA44 , OTA54 */ + { 4.1, 47.6, 1.26}, { 71.6, -6.0, 0.42}, /* OTA64 , OTA74 */ + { -23.4, 46.7, 2.65}, { 31.1, 5.5, -0.64}, /* OTA05 , OTA15 */ + { -43.2, -6.5, 1.68}, { -26.0, -53.2, 0.00}, /* OTA25 , OTA35 */ + { -43.0, 9.1, -0.69}, { 30.1, 69.7, 0.62}, /* OTA45 , OTA55 */ + { -48.5, 61.6, -0.51}, { 65.5, -30.1, 0.69}, /* OTA65 , OTA75 */ + { 2.5, 57.2, 0.17}, { 0.6, 25.4, -0.91}, /* OTA06 , OTA16 */ + { -70.8, -17.4, 1.36}, { 13.6, -24.2, 0.53}, /* OTA26 , OTA36 */ + { -6.8, 28.1, -1.35}, { 20.9, 42.0, 0.19}, /* OTA46 , OTA56 */ + { 6.2, 13.2, -1.50}, { 36.5, 48.8, -0.36}, /* OTA66 , OTA76 */ + { 0.0, 0.0, 0.00}, { 20.4, 1.7, 0.09}, /* OTA07 , OTA17 */ + { 83.8, 11.8, -0.56}, { -26.5, -21.2, 0.56}, /* OTA27 , OTA37 */ + { -70.5, 7.2, -1.05}, { -41.2, -3.6, -1.89}, /* OTA47 , OTA57 */ + { -35.0, 49.5, -0.40}, { 0.0, 0.0, 0.00}}; /* OTA67 , OTA77 */ + +/* Offsets and rotations of each OTA (um, millirad) */ +#ifdef RUN_THREE_V1 +PSC_OFFROT_T psc_otaoff[PSC_NX*PSC_NY]={ /* Offsets and rotations derived o4885 090228 */ { 0.0, 0.0, 0.00}, { 20.2, -59.5, -1.23}, /* OTA00 , OTA10 */ { 28.1,-138.1, 0.18}, { 2.0, -43.6, -1.22}, /* OTA20 , OTA30 */ @@ -61,4 +99,5 @@ { -74.8, 0.3, -0.92}, { -44.6, -1.6, -1.78}, /* OTA47 , OTA57 */ { -50.3, 72.1, -0.47}, { 0.0, 0.0, 0.00}}; /* OTA67 , OTA77 */ +#endif #ifdef RUN_TWO @@ -136,5 +175,5 @@ #endif -static int DOOFFSET; /* Do these offsets or no? */ +static int DOOFFSET=1; /* Do these offsets or no? */ #if 0 @@ -143,5 +182,29 @@ #endif -/* Enable application of chip offsets? */ +/* Enable application of chip offsets (um, mrad)? */ +int psc_load_otaoff(const char *fname) +{ + int i; + char rbuf[1024]; + FILE *fp; + if( (fp=fopen(fname, "r")) == NULL) { + fprintf(stderr, "error: Error opening OTA offset file '%s'\n", fname); + return(-1); + } + for(i=0; i microns notionally */ +/* Offset, scale with quadratic and cubic distortion, rotate */ +int psc_psoptics2(double x, double y, double dx, double dy, double dpa, + double pscale, double d2, double d3, double *xfp, double *yfp) +{ + double w2=((x-dx)*(x-dx)+(y-dy)*(y-dy)), w; + w = sqrt(w2); + x = pscale * (1.0 + d2*w + d3*w2) * (x-dx); + y = pscale * (1.0 + d2*w + d3*w2) * (y-dy); +/* Apply any extra TP-FP rotation */ + *xfp = x * cos(dpa) + y * sin(dpa); + *yfp = -x * sin(dpa) + y * cos(dpa); + return(0); +} + +/* Evaluate an inverse optical model: microns -> arcsec notionally */ +/* Offset, scale with *small* quadratic and cubic distortion, rotate */ +int psc_invoptics2(double xfp, double yfp, double dx, double dy, double dpa, + double pscale, double d2, double d3, double *x, double *y) +{ + double xp, yp, w; +/* Undo any extra TP-FP rotation */ + xp = (xfp * cos(dpa) - yfp * sin(dpa)) / pscale; + yp = (xfp * sin(dpa) + yfp * cos(dpa)) / pscale; + w = sqrt(xp*xp+yp*yp); + *x = xp / (1.0 + d2*w + d3*(xp*xp+yp*yp)) + dx; + *y = yp / (1.0 + d2*w + d3*(xp*xp+yp*yp)) + dy; + return(0); +} + + +/* Evaluate an optical model: arcsec -> microns notionally */ /* Offset, scale with distortion, rotate */ int psc_psoptics(double x, double y, double dx, double dy, double dpa, @@ -318,5 +412,4 @@ return(0); } - /* Project RA,Dec (a,d) to the tangent plane (x,y) at (a0,d0,pa) [radians] */ Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.h =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.h (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.h (revision 24557) @@ -2,21 +2,28 @@ #define NINT(x) (x<0?(int)((x)-0.5):(int)((x)+0.5)) +#define OTA_CTE (5.0e-6) /* Coefficient of thermal expansion of Si+Mo */ +#define CFFP_CTE (0.5e-6) /* Coefficient of thermal expansion of CFFP */ +#define DELTA_T (-100.0) /* Operating temp minus metrology temp */ + +#define OTA_SCALE (1.0+OTA_CTE*DELTA_T) /* Room temp um to operating um of Si+Mo */ +#define CFFP_SCALE ((1.0+CFFP_CTE*DELTA_T)*25400) /* Room temp inches to um on CFFP */ + /* Silicon layout of OTAs (um) */ -#define PSC_PIXEL 10.0 /* Pixel size of an OTA */ -#define PSC_HCELL 5900.0 /* Horizontal cell size */ -#define PSC_VCELL 5980.0 /* Vertical cell size */ -#define PSC_HSTREET 120.0 /* Horizontal street between cells (HS) */ -#define PSC_VSTREET 180.0 /* Vertical street between cells (VS) */ -#define PSC_LBORDER 442.0 /* Left border (to left-most cell) (LB) */ -#define PSC_RBORDER 418.0 /* Right border (to right-most vertical street) (RB) */ -#define PSC_TBORDER 354.5 /* Top border (to top-most cell) (TB) */ -#define PSC_BBORDER 975.5 /* Bottom border (to lowest horizontal street) (BB) */ -#define PSC_HDIE 49500.0 /* Horizontal size of silicon die */ -#define PSC_VDIE 50130.0 /* Vertical size of silicon die */ +#define PSC_PIXEL (10.0*OTA_SCALE) /* Pixel size of an OTA */ +#define PSC_HCELL (5900.0*OTA_SCALE) /* Horizontal cell size */ +#define PSC_VCELL (5980.0*OTA_SCALE) /* Vertical cell size */ +#define PSC_HSTREET (120.0*OTA_SCALE) /* Horizontal street between cells (HS) */ +#define PSC_VSTREET (180.0*OTA_SCALE) /* Vertical street between cells (VS) */ +#define PSC_LBORDER (442.0*OTA_SCALE) /* Left border (to left-most cell) (LB) */ +#define PSC_RBORDER (418.0*OTA_SCALE) /* Right border (to right-most vertical street) (RB) */ +#define PSC_TBORDER (354.5*OTA_SCALE) /* Top border (to top-most cell) (TB) */ +#define PSC_BBORDER (975.5*OTA_SCALE) /* Bottom border (to lowest horizontal street) (BB) */ +#define PSC_HDIE (49500.0*OTA_SCALE) /* Horizontal size of silicon die */ +#define PSC_VDIE (50130.0*OTA_SCALE) /* Vertical size of silicon die */ /* Mechanical layout of OTAs (um) */ -#define PSC_HMECH (1.957*25400) /* Horizontal mech spacing of OTA placement */ -#define PSC_VMECH (2.025*25400) /* Vertical mech spacing of OTA placement */ -#define PSC_VMOFF (PSC_VDIE-(1.568+0.375)*25400) /* Vert die offset between sides */ +#define PSC_HMECH (1.957*CFFP_SCALE) /* Horizontal mech spacing of OTA placement */ +#define PSC_VMECH (2.025*CFFP_SCALE) /* Vertical mech spacing of OTA placement */ +#define PSC_VMOFF (PSC_VDIE-(1.568+0.375)*CFFP_SCALE) /* Vert die offset between sides */ #define PSC_REFRACT_CONST 60.0 /* Standard refraction ("/tanz) at STP */ @@ -123,11 +130,22 @@ /* Overall focal plane offset and rotation (um, rad) */ -const PSC_OFFROT_T psc_fpoff; +PSC_OFFROT_T psc_fpoff; /* Offsets and rotations of each OTA (um, millirad) */ -const PSC_OFFROT_T psc_otaoff[PSC_NX*PSC_NY]; - +PSC_OFFROT_T psc_otaoff[PSC_NX*PSC_NY]; + + +/* This is from Run 3, 090516 */ +#define PS_scale 38.7932 /* Default PS plate scale [um/arcsec] */ +#define PS_d2 9.78e-7 /* Default quadratic PS distortion [arcsec^-1] */ +#define PS_d3 3.16e-11 /* Default cubic PS distortion [arcsec^-2] */ + + +#ifdef RUN_THREE_V1 /* This is from Run 3, 090228 */ #define PS_scale 38.860 /* Default PS plate scale [um/arcsec] */ -#define PS_d3 1.49e-10 /* Default PS distortion [arcsec^-2] */ +#define PS_d3 1.49e-10 /* Default cubic PS distortion [arcsec^-2] */ +#endif + + #define PS_airdens 0.71 /* Default PS1 qir density (Haleakala) */ @@ -139,7 +157,11 @@ * --------------------------------------------------------------------------- * psc_tproject(a, d, a0, d0, pa, density, pi/2-alt, vertical, &x, &y); - * psc_psoptics(x/sec2rad, y/sec2rad, dx, dy, dpa, pscale, d3, &xfp, &yfp); + * psc_psoptics2(x/sec2rad, y/sec2rad, dx, dy, dpa, pscale, d2, d3, &xfp, &yfp); * psc_fp_to_pixel(xfp, yfp, &ota_xid, &ota_yid, &xota, &yota); * psc_pixel_to_cell(xota, yota, &cell_xid, &cell_yid, &xcell, &ycell); + * + * can also use + * psc_psoptics(x/sec2rad, y/sec2rad, dx, dy, dpa, pscale, d3, &xfp, &yfp); + * * * To go from cell coords to a position (a,d) relative to (a0,d0,pa) pointing: @@ -147,6 +169,9 @@ * psc_cell_to_pixel(cell_xid,cell_yid, xcell,ycell, &xota,&yota); * psc_pixel_to_fp(ota_xid, ota_yid, xota, yota, &xfp, &yfp); + * psc_invoptics2(xfp, yfp, dx, dy, dpa, pscale, d2, d3, &x, &y); + * psc_tplonglat(x*sec2rad, y*sec2rad, a0, d0, pa, density, pi/2-alt, vertical, &a, &d); + * + * can also use * psc_invoptics(xfp, yfp, dx, dy, dpa, pscale, d3, &x, &y); - * psc_tplonglat(x*sec2rad, y*sec2rad, a0, d0, pa, density, pi/2-alt, vertical, &a, &d); */ @@ -155,5 +180,5 @@ /**************/ /* Enable application of chip offsets? */ -int psc_do_chipoff(int doit); /* Apply chip offsets? (0/1, default=0) */ +int psc_do_chipoff(int doit); /* Apply chip offsets? (0/1, default=1) */ /* Convert a focal plane position to OTA otax,otay, Cell cellx,celly, Pixel */ @@ -202,2 +227,19 @@ double pscale/*38.86um/sec*/, double d3/*1.49e-10sec^-2*/, double *xfp_um, double *yfp_um); + +/* Evaluate an inverse optical model: microns -> arcsec notionally */ +/* Offset, scale with *small* distortion, rotate */ +int psc_invoptics2(double xfp_um, double yfp_um, double dx_sec, double dy_sec, double dpa_rad, + double pscale/*38.793um/sec*/, + double d2/*9.8e-7sec^-2*/, double d3/*3.1e-11sec^-2*/, + double *x_sec, double *y_sec); + +/* Evaluate an optical model: arcsec -> microns notionally */ +/* Offset, scale with distortion, rotate */ +int psc_psoptics2(double x_sec, double y_sec, double dx_sec, double dy_sec, double dpa_rad, + double pscale/*38.793um/sec*/, + double d2/*9.8e-7sec^-2*/, double d3/*3.1e-11sec^-2*/, + double *xfp_um, double *yfp_um); + +/* Enable application of chip offsets (um, mrad)? */ +int psc_load_otaoff(const char *fname); Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burnfix.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burnfix.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burnfix.c (revision 24557) @@ -105,4 +105,19 @@ if(!cell->persist[k].fiterr) sub_fit(nx, ny, NX, buf, &(cell->persist[k]), -1); + } + return(0); +} + + +/****************************************************************/ +/* burn_apply(): Subtract the trail fits from the image */ +STATIC int burn_apply(int nx, int ny, int NX, IMTYPE *buf, CELL *cell) +{ + int k; + +/* Restore all the burns */ + for(k=0; knpersist; k++) { + if(!cell->persist[k].fiterr) + sub_fit(nx, ny, NX, buf, &(cell->persist[k]), +1); } return(0); Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c (revision 24557) @@ -33,5 +33,5 @@ int otanum; int nextend, cellxy, cell, cellcode; - int ext, update, restore, psfsize, psfavg; + int ext, update, restore, apply, tableonly, psfsize, psfavg; IMTYPE *buf; const char *burnfile=NULL, *persistfile=NULL, *persistfitsfile=NULL; @@ -102,6 +102,8 @@ /* Parse the args */ cellxy = -1; - update = 1; - restore = 0; + update = 1; /* Calc fits, apply fits, write img and table */ + restore = 0; /* Restore previous fit only, write img */ + apply = 0; /* Apply previous fit only, write img */ + tableonly = 0; /* Calc fits (apply fits), write table only */ psfsize = 32; psfavg = 0; @@ -137,7 +139,15 @@ update = argv[i][7] == 'y' || argv[i][7] == '1' || argv[i][7] == 't'; -/* Modify the input MEF by subtracting fits? */ +/* Modify the input MEF by adding back fits? */ } else if(strncmp(argv[i], "restore=", 8) == 0) { /* restore={t|f} */ restore = argv[i][8] == 'y' || argv[i][8] == '1' || argv[i][8] == 't'; + +/* Modify the input MEF by subtracting previously calculated fits? */ + } else if(strncmp(argv[i], "apply=", 6) == 0) { /* apply={t|f} */ + apply = argv[i][6] == 'y' || argv[i][6] == '1' || argv[i][6] == 't'; + +/* Calculate and write tables only? */ + } else if(strncmp(argv[i], "tableonly=", 10) == 0) {/* tableonly={t|f} */ + tableonly = argv[i][10] == 'y' || argv[i][10] == '1' || argv[i][10] == 't'; /* Output file for burn streaks */ @@ -150,9 +160,9 @@ /* Input text file for previous burn persistence streaks */ - } else if(strncmp(argv[i], "infits=", 8) == 0) { /* in=fname */ + } else if(strncmp(argv[i], "trailin=", 8) == 0) {/* trailin=fname */ persistfile = argv[i] + 8; /* Same thing, but information is stored in tables in a FITS file. */ - } else if(strncmp(argv[i], "trailinfits=", 8) == 0) { /* infits=fname */ + } else if(strncmp(argv[i], "trailinfits=", 12) == 0) { /* trailinfits=fname */ persistfitsfile = argv[i] + 12; @@ -162,6 +172,6 @@ /* Same thing, but information is stored in tables in a FITS file. */ - } else if(strncmp(argv[i], "trailinfits=", 8) == 0) { /* trailin=fname */ - persistfitsfile = argv[i] + 12; + } else if(strncmp(argv[i], "infits=", 7) == 0) { /* infits=fname */ + persistfitsfile = argv[i] + 7; /* Output file for PSF gallery */ @@ -322,6 +332,6 @@ /* If there is no other persistence info supplied, try getting * it from the input FITS file. */ - if(restore && persistfile == NULL) { - if(persistfitsfile == NULL) persistfitsfile = ifilename; + if((restore || apply) && persistfile == NULL) { + if(persistfitsfile == NULL) persistfitsfile = ifilename; } @@ -334,9 +344,9 @@ /* Read the persistence data for this OTA */ - if(persistfile != NULL) { + if(persistfile != NULL) { /* Text data file */ if(persist_read(OTA, persistfile)) exit(EXIT_FAILURE); - } - else if(persistfitsfile != NULL) { - if(persist_fits_read(OTA, persistfitsfile) != FH_SUCCESS) exit(EXIT_FAILURE); + } else if(persistfitsfile != NULL) { /* FITS table */ + if(persist_fits_read(OTA, persistfitsfile) != FH_SUCCESS) + exit(EXIT_FAILURE); } @@ -369,5 +379,5 @@ "warning: Restoring old burns, but header indicates no burns previously corrected.\n"); } - else if (update && (burn_applied == FH_TRUE)) { + else if ((update||apply) && (burn_applied == FH_TRUE)) { fprintf(stderr, "warning: Applying burn correction, but header indicates burns previously corrected.\n"); @@ -491,5 +501,5 @@ if(VERBOSE & VERB_NORM) { - printf("nx=%d ny=%d prex=%d postx=%d posty=%d BZERO=%.1f\n", + printf("nx=%d ny=%d prex=%d postx=%d posty=%d BZERO=%d\n", naxis1, naxis2, prescan1, ovrscan1, ovrscan2, BZERO); } @@ -542,7 +552,17 @@ } - - if(!restore) { - + if(apply) { +/* Use the table-driven fits instead of calculating new ones */ + burn_apply(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, + buf, OTA+cell); +/* Tell us about it? */ + if(VERBOSE & VERB_NORM) burn_blab(OTA+cell); + + } else if(restore) { +/* Restore the old burns */ + burn_restore(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, + buf, OTA+cell); + + } else { /* Fix up the burns */ burn_fix(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, naxis2, buf, @@ -559,19 +579,15 @@ if(VERBOSE & VERB_NORM) burn_blab(OTA+cell); -/* Fix up the streaks */ - persist_fix(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, buf, - OTA+cell); - +/* Fix up the streaks (don't bother if table only) */ + if(!tableonly) { + persist_fix(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, buf, + OTA+cell); /* Tell us about it? */ - if(VERBOSE & VERB_NORM) persist_blab(OTA+cell); - - } else { -/* Restore the old burns */ - burn_restore(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, - buf, OTA+cell); + if(VERBOSE & VERB_NORM) persist_blab(OTA+cell); + } } /* Write the corrected data back to the FITS. */ - if(update) { + if(!tableonly && update) { fh_ehu(ehu, 0); /* Seek back to the start of data */ if (fh_write_padded_image(ehu, fh_file_desc(ehu), buf, @@ -590,10 +606,10 @@ /* Dump out the postage stamp file */ - if(psffile != NULL) { + if(psffile != NULL && !restore && !apply) { psf_write(psfsize, psfsize, OTA, otanum, psffile); } /* Dump out the PSF stats */ - if(psfstatfile != NULL) { + if(psfstatfile != NULL && !restore && !apply) { psf_write_stats(psfsize, psfsize, OTA, otanum, psfstatfile, psfavg); } @@ -601,11 +617,15 @@ /* Write burn info to FITS file. */ if(update) persist_fits_write(OTA, ihu); - - if(restore) { + if(restore || tableonly) { /* Indicate in the header that the burns are not applied. */ - fh_set_bool(ihu, FH_AUTO, PHU_NAME_BURN_APPLIED, - FH_FALSE, PHU_COMMENT_BURN_APPLIED); - fh_rewrite(ihu); + fh_set_bool(ihu, FH_AUTO, PHU_NAME_BURN_APPLIED, + FH_FALSE, PHU_COMMENT_BURN_APPLIED); + fh_rewrite(ihu); + } else if(apply) { + /* Indicate in the header that the burns have been applied. */ + fh_set_bool(ihu, FH_AUTO, PHU_NAME_BURN_APPLIED, + FH_TRUE, PHU_COMMENT_BURN_APPLIED); + fh_rewrite(ihu); } @@ -711,6 +731,8 @@ printf(" cell=N Work on just one cell? Cell count [0:63] mode.\n"); printf(" mask=0101... 64 digits to work on cells 0:63.\n"); - printf(" update={t|f} Modify the input MEF by subtracting fits?\n"); + printf(" update={t|f} Modify the input MEF writing table and subtracting fits?\n"); printf(" restore={t|f} Restore the input MEF by adding input fits?\n"); + printf(" apply={t|f} Modify the input MEF by subtracting previously calculated fits?\n"); + printf(" tableonly={t|f} Calculate fits but do *not* modify the input MEF images, only write tables\n"); printf(" in=fname Input file for previous burn persistence streaks\n"); printf(" infits=fname Input FITS file for previous burn persistence streaks (stored\n"); Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.h =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.h (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.h (revision 24557) @@ -173,4 +173,5 @@ MTYPE *mask, OBJBOX *box); STATIC int burn_restore(int nx, int ny, int NX, IMTYPE *buf, CELL *cell); +STATIC int burn_apply(int nx, int ny, int NX, IMTYPE *buf, CELL *cell); STATIC int persist_read(CELL *cell, const char *infile); STATIC int persist_write(CELL *cell, const char *outfile); Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1 (revision 24557) @@ -66,7 +66,54 @@ inhibited from subtracting the fits by "update=f". + Two other modes can be invoked using "tableonly=t" and "apply=t". + The first mode is just like "update=t", except that it does not write + the modified images, only the FITS table and/or tabulated fit data. + The second mode is like "restore=t", except that it subtracts the + tabulated fits instead of adding them back in. Note that this requires + slightly different specifications for the "in=" file: + + update or tableonly: in=previous_file_for_persistence + restore or apply: in=this_file_for_burn_and_persistence + + Thus a sequence of images could be processed (leaving off usual + arguments such as mask=, etc) to immediately fit and subtract: + + burntool o4991g0001o24.fits \ + out=o4991g0001o24.burn \ + psfstat=o4991g0001o24.stat psf=o4991g0001o24.psf + + burntool o4991g0002o24.fits \ + in=o4991g0001o24.burn out=o4991g0002o24.burn \ + psfstat=o4991g0002o24.stat psf=o4991g0002o24.psf + + burntool o4991g0003o24.fits \ + in=o4991g0002o24.burn out=o4991g0003o24.burn \ + psfstat=o4991g0003o24.stat psf=o4991g0003o24.psf + + or as a two step process of generating tables: + + burntool o4991g0001o24.fits tableonly=t \ + out=o4991g0001o24.burn \ + psfstat=o4991g0001o24.stat psf=o4991g0001o24.psf + + burntool o4991g0002o24.fits tableonly=t \ + in=o4991g0001o24.burn out=o4991g0002o24.burn \ + psfstat=o4991g0002o24.stat psf=o4991g0002o24.psf + + burntool o4991g0003o24.fits tableonly=t \ + in=o4991g0002o24.burn out=o4991g0003o24.burn \ + psfstat=o4991g0003o24.stat psf=o4991g0003o24.psf + + and then later on carrying out the subtractions of the fits: + + burntool o4991g0001o24.fits apply=t + burntool o4991g0002o24.fits apply=t + burntool o4991g0003o24.fits apply=t + The identification of significant (but unsaturated) stars is an important component of burntool's mask generation, and if requested burntool can assemble a gallery of postage stamps of suitable stars. + This does not work with "restore=t" or "apply=t" of course, since + these work only on previous tables and do not examine the images. Any star whose maximum exceeds PSF_THRESH above sky but is not deemed @@ -125,7 +172,16 @@ qt=-5.201 Tangential quadrupole [pix^2] (-99.99 if fails) q+ * cos(2*phi) + qx * sin(2*phi) + q3c=-1.193 Cosine trefoil measure [pix^2] (-99.99 if fails) + q3s=-0.038 Sine trefoil measure [pix^2] (-99.99 if fails) qpm=5.154 Plus quadrupole averaged over psfavg qcm=0.560 Cross quadrupole averaged over psfavg qtm=-5.201 Tangential averaged over psfavg + q3cm=-1.230 Cosine trefoil averaged over psfavg + q3sm=-0.079 Sine trefoil averaged over psfavg + + The net, polar coordinate quadrupole can be assembled from + + Qmagnitude = sqrt(qpm^2+qcm^2) + Qtheta = 0.5 * atan2(qcm, qpm) Helpful utilities include: @@ -151,8 +207,18 @@ update={t|f} - Modify the input MEF by subtracting fits (default t)? + Calculate fits and modify the input MEF images by subtracting + the trail fits and writing a FITS table (default t). If + false the MEF is not altered, although PSF can be calculated. restore={t|f} - Restore the input MEF by adding input fits (default f)? + Restore the input MEF by adding input fits (default f) + + apply={t|f} + Use previously calculated fits to modify the input MEF image + by subtraction (default f) + + tableonly={t|f} + Calculate fits but do *not* modify the input MEF images, + only write the fit data as a FITS and/or text table (default f) in=fname Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/psfstamp.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/psfstamp.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/psfstamp.c (revision 24557) @@ -136,5 +136,5 @@ int i, k, l, nstar, fdout, otacx, otacy, xid, yid, ntot=0, sumax; int cellcount, ota_xid, ota_yid; - double scale, phi, fwhm[3], q[3], qt, xfp, yfp, pi=4*atan(1.0); + double scale, phi, fwhm[3], q[5], qt, xfp, yfp, pi=4*atan(1.0); IMTYPE *median_image; CELL *cell; @@ -203,6 +203,7 @@ qt = -q[1] * cos(2*phi) - q[2] * sin(2*phi); - printf("N= %d PSFmaj= %.2f min= %.2f theta= %.1f m2= %.2f q+= %.3f qx= %.3f qt= %.3f\n", - nstar, fwhm[0], fwhm[1], fwhm[2]*180/pi, q[0], q[1], q[2], qt); + printf("N= %d PSFmaj= %.2f min= %.2f theta= %.1f m2= %.2f q+= %.3f qx= %.3f qt= %.3f q3c= %.3f q3s= %.3f\n", + nstar, fwhm[0], fwhm[1], fwhm[2]*180/pi, q[0], q[1], q[2], qt, + q[3], q[4]); if(CONCAT_FITS) { @@ -258,9 +259,12 @@ double m2[MAXPSFMEDIAN], qp[MAXPSFMEDIAN], qc[MAXPSFMEDIAN]; double qt[MAXPSFMEDIAN], fwavg[MAXPSFMEDIAN]; + double q3c[MAXPSFMEDIAN], q3s[MAXPSFMEDIAN]; double qpavg[MAXPSFMEDIAN], qcavg[MAXPSFMEDIAN], qtavg[MAXPSFMEDIAN]; + double q3cavg[MAXPSFMEDIAN], q3savg[MAXPSFMEDIAN]; int nstar[MAXCELL], nfw[MAXCELL]; double fwmed[MAXCELL], m2med[MAXCELL]; double qpmed[MAXCELL], qcmed[MAXCELL], qtmed[MAXCELL]; - double qpmacro, qcmacro, qtmacro, fwmacro; + double q3cmed[MAXCELL], q3smed[MAXCELL]; + double qpmacro, qcmacro, qtmacro, fwmacro, q3cmacro, q3smacro; FILE *fp; CELL *cell; @@ -304,4 +308,6 @@ qp[nstar[k]] = q[1]; qc[nstar[k]] = q[2]; + q3c[nstar[k]] = q[3]; + q3s[nstar[k]] = q[4]; /* Get the position in the focal plane and therefore the qt statistic */ psc_cell_to_pixel(cellx, celly, 0.5*PSC_HCELL/PSC_PIXEL, @@ -325,6 +331,9 @@ qcmed[k] = double_median(nstar[k], qc); qtmed[k] = double_median(nstar[k], qt); + q3cmed[k] = double_median(nstar[k], q3c); + q3smed[k] = double_median(nstar[k], q3s); } else { m2med[k] = qpmed[k] = qcmed[k] = qtmed[k] = -99.99; + q3cmed[k] = q3smed[k] = -99.99; } /* Toss these results into the macrocell median hopper */ @@ -337,4 +346,6 @@ qcavg[nqavg] = qc[l]; qtavg[nqavg] = qt[l]; + q3cavg[nqavg] = q3c[l]; + q3savg[nqavg] = q3s[l]; nqavg++; } @@ -346,5 +357,7 @@ qcmacro = double_median(nqavg, qcavg); qtmacro = double_median(nqavg, qtavg); - if(nqavg == 0) qpmacro = qcmacro = qtmacro = -99.99; + q3cmacro = double_median(nqavg, q3cavg); + q3smacro = double_median(nqavg, q3savg); + if(nqavg == 0) qpmacro = qcmacro = qtmacro = q3cmacro = q3smacro = -99.99; fwmacro = double_median(nfwave, fwavg); @@ -357,9 +370,9 @@ cell = ota + k; - fprintf(fp, "ext=xy%1d%1d bias=%d sky=%d rmssky=%d npsf=%d fwhm=%.2f fwmed=%.2f m2=%.2f qp=%.3f qc=%.3f qt=%.3f qpm=%.3f qcm=%.3f qtm=%.3f\n", + fprintf(fp, "ext=xy%1d%1d bias=%d sky=%d rmssky=%d npsf=%d fwhm=%.2f fwmed=%.2f m2=%.2f qp=%.3f qc=%.3f qt=%.3f q3c=%.3f q3s=%.3f qpm=%.3f qcm=%.3f qtm=%.3f q3cm=%.3f q3sm=%.3f\n", cellx, celly, cell->bias, cell->sky, cell->rms, nstar[k], fwmed[k], fwmacro, m2med[k], - qpmed[k], qcmed[k], qtmed[k], - qpmacro, qcmacro, qtmacro); + qpmed[k], qcmed[k], qtmed[k], q3cmed[k], q3smed[k], + qpmacro, qcmacro, qtmacro, q3cmacro, q3smacro); } } Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/psfstats.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/psfstats.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/burntool/psfstats.c (revision 24557) @@ -10,10 +10,9 @@ #include "burntool.h" #include "math.h" -#include "psf/psf.h" /****************************************************************/ /* psf_stats(): Tell us about this PSF star */ -STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias, - double *fwhm, double *q) +STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias, + double *fwhm, double *q) { return(1); Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfh/fh.c (revision 24557) @@ -200,5 +200,5 @@ /* Functions that do file operations WITH RETRY: */ static int write_file(int fd, const void* buf, int len); -static int read_file(int fd, const void* buf, int len); +static int read_file(int fd, void* buf, int len); static int seek_file(int fd, off_t off); /* Performs lseek(SEEK_SET) */ static int fh_lock_file(HeaderUnit hu, int fd, fh_mode mode); @@ -234,4 +234,5 @@ * this information in debug mode. */ +const char* fh_rcs_version(void); /* proto to make gcc happy */ const char* fh_rcs_version(void) { return rcs_id; } @@ -274,5 +275,5 @@ static void -null_log(const char* s) { if (s) ; } +null_log(const char* s) { if (s) return; } /* * It's also possible to pass a NULL to fh_log_xxx() and rely only on the @@ -309,5 +310,5 @@ HeaderUnit -fh_create() +fh_create(void) { HeaderUnitStruct* rtn; @@ -1941,5 +1942,6 @@ HeaderUnitStruct* list = FH_HU(hu); char writebuf[FH_BLOCK_SIZE]; - int i = 0, writelen; + int i = 0; + unsigned int writelen; int blocks_left; fh_result result; @@ -2446,5 +2448,5 @@ else /* byte-swapping case requires a temporary buffer */ { - unsigned char outbuf[32768]; + unsigned char* outbuf[32768]; while (bytes_left) @@ -2811,5 +2813,32 @@ while (len) { - rtn = write(fd, (char*)buf, len); + rtn = write(fd, (const char*)buf, len); + + switch (rtn) + { + case -1: if (errno == EINTR || errno == EAGAIN) + continue; /* retry */ + return -1; /* permanent failure */ + case 0: return count; + default: + { + len -= rtn; + count += rtn; + buf = (const char*)buf + rtn; + } + } + } + return count; +} + +static int +read_file(int fd, void* buf, int len) +{ + int rtn; + int count = 0; + + while (len) + { + rtn = read(fd, (char*)buf, len); switch (rtn) @@ -2831,31 +2860,4 @@ static int -read_file(int fd, const void* buf, int len) -{ - int rtn; - int count = 0; - - while (len) - { - rtn = read(fd, (char*)buf, len); - - switch (rtn) - { - case -1: if (errno == EINTR || errno == EAGAIN) - continue; /* retry */ - return -1; /* permanent failure */ - case 0: return count; - default: - { - len -= rtn; - count += rtn; - buf = (char*)buf + rtn; - } - } - } - return count; -} - -static int seek_file(int fd, off_t off) { @@ -2969,5 +2971,5 @@ fh_compare(const void* a, const void* b) { - double diff = (*((FitsCard**)a))->idx - (*((FitsCard**)b))->idx; + double diff = (*((FitsCard* const*)a))->idx - (*((FitsCard* const*)b))->idx; if (diff < 0) return -1; Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfh/fh_table.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfh/fh_table.c (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfh/fh_table.c (revision 24557) @@ -180,8 +180,10 @@ /* Write the format specifiers for each column. */ - for(i=0; i < table->num_cols; i++) - { + for (i=0; i < table->num_cols; i++) + { + int n; + idx = 5.0 + i/10.; - int n = i + 1; /* FITS numbering scheme starts from 1. */ + n = i + 1; /* FITS numbering scheme starts from 1. */ snprintf(keyword, FH_NAME_SIZE + 1, "TFORM%d", n); if((table->cols[i].format == FH_TABLE_FORMAT_FLOAT) || @@ -214,8 +216,10 @@ * nice to have from a self-documenting perspective, isn't strictly * necessary to interpret the actual table values. */ - for(i=0; i < table->num_cols; i++) - { + for (i=0; i < table->num_cols; i++) + { + int n; + idx = 7.0 + i/10.; - int n = i + 1; /* FITS numbering scheme starts from 1. */ + n = i + 1; /* FITS numbering scheme starts from 1. */ snprintf(keyword, FH_NAME_SIZE + 1, "TTYPE%d", n); fh_set_str(hu, idx, keyword, table->cols[i].name, Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfhreg/gpc_telescope.h =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfhreg/gpc_telescope.h (revision 24529) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/libfhreg/gpc_telescope.h (revision 24557) @@ -89,4 +89,5 @@ MK_PFL( 523.3, COMALT ,6, "Commanded telescope pointing altitude (degrees)" ) MK_PFL( 523.4, COMROT ,6, "Commanded telescope rotator angle (degrees)" ) +MK_PFL( 523.5, MOONANG ,6, "Angular distance to moon (degrees)" ) MK_CMT( 530.00,CMTTELO1, "NOTE: Telescope RA DEC or ALT AZ already include") MK_CMT( 530.10,CMTTELO2, " the following offsets. Do not re-apply!" ) Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/Make.Common =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/Make.Common (revision 24557) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/Make.Common (revision 24557) @@ -0,0 +1,1 @@ +link ../Make.Common Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/Makefile =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/Makefile (revision 24557) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/Makefile (revision 24557) @@ -0,0 +1,16 @@ +include ../Make.Common + +VERSION=0.00 +CCWARN+=$(WERROR) +CCDEFS+=$(VERSIONDEFS) +$(EXECNAME): $(OBJS) libfh.a + +include ../Make.Common + +# Dependencies by Make.Common $Revision: 2.17 $ + +$(OBJ)/sort.o: sort.c + +$(OBJ)/tiltystreak.o: tiltystreak.c fh/fh.h \ + fhreg/general.h fhreg/macros.h \ + fhreg/gpc_detector.h Index: branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/linearfit.c =================================================================== --- branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/linearfit.c (revision 24557) +++ branches/eam_branches/20090522/extsrc/gpcsw/gpcsrc/fits/tiltystreak/linearfit.c (revision 24557) @@ -0,0 +1,129 @@ +/* Routine to fit a linear function */ +/* 060304 v1.2 add wlinearfit() */ +/* 060130 v1.1 replaced static allocations with calloc's */ +/* 040208 v1.0 John Tonry (derived from fortran) */ +#include +#include +#include + +#define ABS(x) ((x)<0?(-(x)):(x)) + +static int linsolve(int n, double *y, double *a, double *x); + +int linearfit(int npt, double *y, int nx, double *x, double *param) +{ +/* Y = NPT data being fitted as a linear function of X[*][NX] + * PARAM = returned parameters of fit: + * Y[*] = PARAM[0]*X[*][0], + * + PARAM[1]*X[*][1], + * ... + * + PARAM[NX-1]*X[*][NX-1] + * + */ + int i, j, k; + double *v, *a; + v = (double *)calloc(nx, sizeof(double)); + a = (double *)calloc(nx*nx, sizeof(double)); + + if(npt < nx) { + fprintf(stderr, "linearfit: Too few points provided: %d %d\n", npt, nx); + return(-2); + } + +#if 0 /* Show us data to solve? */ + for(i=0; i rat) { + k = i; + rat = ABS(a[j+k*n]); + } + } + rowstatus[k] = 1; + row[j] = k; + if(rat == 0.0) { + fprintf(stderr, "WHOA: singular matrix, iter %d, row %d\n", j, k); + free(rowstatus); + free(row); + return(1); + } +/* Subtract away matrix below diagonal */ + for(i=0; i=0; j--) { + x[j] = y[row[j]]; + for(k=j+1; k +#include +#include + +#define MAXSTACK 256 +//#define NSTOP 15 +#define NSTOP 8 + +/* Quicksort program 1980 (John Tonry) */ +int qsort_dbl(int n, double *x) +{ + double key, kl, kr, km, temp; + int l, r, m, lstack[MAXSTACK], rstack[MAXSTACK], sp; + register int i, j, k; + int mgtl, lgtr, rgtm; + + sp = 0; + lstack[sp] = 0; + rstack[sp] = n-1; + + while(n > NSTOP && sp >= 0) { +/* Sort a subrecord off the stack */ + l = lstack[sp]; + r = rstack[sp]; + sp--; + m = (l + r) / 2; +/* Set KEY = median of X(L), X(M), X(R) */ + kl = x[l]; + km = x[m]; + kr = x[r]; + mgtl = km > kl; + rgtm = kr > km; + lgtr = kl > kr; +#ifndef STD_KEY + if(mgtl ^ rgtm) { + if(mgtl ^ lgtr) key = kr; + else key = kl; + } else { + key = km; + } +#else +/* Curiously enough, this non-median seems to work as well or better */ + if(mgtl ^ rgtm) { + key = km; + } else { + if(mgtl ^ lgtr) key = kl; + else key = kr; + } +#endif + i = l; + j = r; + while(1) { +/* Find a big record on the left */ + while(x[i] < key) i++; + +/* Find a small record on the right */ + while(x[j] > key) j--; + + if(i >= j) break; +/* Exchange records */ + temp = x[i]; + x[i] = x[j]; + x[j] = temp; + i++; + j--; + } + +/* Subfile is partitioned into two halves, left .le. right */ +/* Push the two halves on the stack */ + if(j-l+1 > NSTOP) { + sp = sp + 1; + lstack[sp] = l; + rstack[sp] = j; + } + if(r-j > NSTOP) { + sp = sp + 1; + lstack[sp] = j+1; + rstack[sp] = r; + } + if(sp >= MAXSTACK) { +// fprintf(stderr,"QSORT8: Fatal error from stack overflow\n"); +// fprintf(stderr,"Fall back on sort by insertion\n"); + break; + } + } + +/* Sorting routine that sorts the N elements of single precision */ +/* array X by straight insertion between previously sorted numbers */ + for(j=n-2; j>=0; j--) { + for(i=j+1, k=j; i NSTOP && sp >= 0) { +/* Sort a subrecord off the stack */ + l = lstack[sp]; + r = rstack[sp]; + sp--; + m = (l + r) / 2; +/* Set KEY = median of X(L), X(M), X(R) */ + kl = x[l]; + km = x[m]; + kr = x[r]; + mgtl = km > kl; + rgtm = kr > km; + lgtr = kl > kr; +#ifndef STD_KEY + if(mgtl ^ rgtm) { + if(mgtl ^ lgtr) key = kr; + else key = kl; + } else { + key = km; + } +#else +/* Curiously enough, this non-median seems to work as well or better */ + if(mgtl ^ rgtm) { + key = km; + } else { + if(mgtl ^ lgtr) key = kl; + else key = kr; + } +#endif + i = l; + j = r; + while(1) { +/* Find a big record on the left */ + while(x[i] < key) i++; + +/* Find a small record on the right */ + while(x[j] > key) j--; + + if(i >= j) break; +/* Exchange records */ + temp = x[i]; + x[i] = x[j]; + x[j] = temp; + i++; + j--; + } + +/* Subfile is partitioned into two halves, left .le. right */ +/* Push the two halves on the stack */ + if(j-l+1 > NSTOP) { + sp = sp + 1; + lstack[sp] = l; + rstack[sp] = j; + } + if(r-j > NSTOP) { + sp = sp + 1; + lstack[sp] = j+1; + rstack[sp] = r; + } + if(sp >= MAXSTACK) { +// fprintf(stderr,"QSORT8: Fatal error from stack overflow\n"); +// fprintf(stderr,"Fall back on sort by insertion\n"); + break; + } + } + +/* Sorting routine that sorts the N elements of single precision */ +/* array X by straight insertion between previously sorted numbers */ + for(j=n-2; j>=0; j--) { + for(i=j+1, k=j; i +#include +#include + +#include "math.h" +#include "fh/fh.h" +#include "fhreg/general.h" +#include "fhreg/gpc_detector.h" + +#define MAX(a,b) (((a) > (b)) ? (a) : (b)) +#define MIN(a,b) (((a) < (b)) ? (a) : (b)) +#define NINT(x) (x<0?(int)((x)-0.5):(int)((x)+0.5)) +#define ABS(a) (((a) > 0) ? (a) : -(a)) + +#define NODATA 0 /* Marker for *No Data* */ +#define NCELL 8 /* Number of cells in a row */ +#define SAT4SURE 45000 /* Blasted, we don't care about pixels above */ +#define NPOLY 3 /* N terms in of flattening polynomial */ + +//#define DEBUG /* Turn on debug output? */ + +/* Info for a cell */ +typedef struct cell_info { + HeaderUnit hu; /* Header unit for this cell */ + int cell; /* Cell number */ + int mefxt; /* Where is it in the MEF? */ + int bias; /* Bias level */ + int sky; /* Sky level */ + int rms; /* RMS in the sky */ + int nsat; /* Saturated pixel count */ + double ampl; /* Streak amplitude */ + short *data; /* Image data */ + int *mask; /* Mask bits */ +} CELL; + +/* Prototypes */ +static int cell_stats(int sx, int nx, int ny, int NX, int NY, int bzero, + CELL *cell); +static int int_median(int n, int *key); +int qsort_dbl(int n, double *x); +int qsort_int(int n, int *x); +static int streak_ratio(int sx, int nx, int ny, int NX, CELL otarow[]); +static int linear_streak_fit(int sx, int nx, int ny, int NX, CELL otarow[]); +static int bilinear_streak_fit(int sx, int nx, int ny, int NX, CELL otarow[]); +static int parabola_streak_fit(int sx, int nx, int ny, int NX, CELL otarow[]); +int linearfit(int npt, double *y, int nx, double *x, double *param); +static void syntax(const char *prog); + +static int nmedian_buf=0; +static int *median_buf=NULL; +static double *Abuf=NULL, *Bbuf=NULL, *Xbuf=NULL; + +int main(int argc, const char* argv[]) +{ + int i, j, row, col, ext; + HeaderUnit ihu = fh_create(); + HeaderUnit ehu = NULL; + const char* ifilename; + char extname[FH_MAX_STRLEN+1], xtension[FH_MAX_STRLEN+1], cellsought[5]; + int nextend; + CELL otarow[NCELL]; + int naxis, naxis1=0, naxis2=0; + int n1, n2; + int prescan1, ovrscan1, ovrscan2; + double bzero_d; + int bzero = 32768, firstime = 1, algo=2; + int cellmask[NCELL*NCELL]; + + if(argc < 2) { + syntax(argv[0]); + fprintf(stderr, "error: syntax: tiltystreak Uncompressed_GPC1_MEF.fits\n"); + exit(EXIT_FAILURE); + } + ifilename = argv[1]; + +/* Defaults */ + for(i=0; i nextend) { + fprintf(stderr, + "error: Could not find extension `%s'\n", cellsought); + exit(EXIT_FAILURE); + } + +/* Positioned at the desired cell now */ + if (fh_get_PRESCAN1(ehu, &prescan1) != FH_SUCCESS || + fh_get_OVRSCAN1(ehu, &ovrscan1) != FH_SUCCESS || + fh_get_OVRSCAN2(ehu, &ovrscan2) != FH_SUCCESS || + fh_get_NAXIS(ehu, &naxis) != FH_SUCCESS || + fh_get_NAXIS1(ehu, &n1) != FH_SUCCESS || + fh_get_NAXIS2(ehu, &n2) != FH_SUCCESS) { + fprintf(stderr, "error: Cannot get NAXIS* in extension %d row %d col %d\n", ext, row, col); + exit(EXIT_FAILURE); + } +/* Cell isn't an image or is 32bpp */ + if(naxis != 2) { + otarow[col].mefxt = -1; + continue; + } + +/* Make some space now that we finally know the size */ + if(firstime) { + naxis1 = n1; + naxis2 = n2; + firstime = 0; + for(i=0; i 0) i++; + if(i == 0) continue; /* Nope, entirely wiped out */ + +/* Process this row of cells */ + +/* What's the stats in each cell, generate mask */ + for(col=0; colbias = 0; + } else { + for(j=1; jdata[sx+nx+(NX-nx-sx)/2+j*NX] + bzero; + cell->bias = int_median(ny-2, median_buf); + } + +/* Get sky stats */ + for(k=n=0; kdata[sx+i+NX*j]+bzero < SAT4SURE && + cell->data[sx+i+NX*j]+bzero != NODATA) { + median_buf[n++] = cell->data[sx+i+NX*j] + bzero; + } + } + + if(n < 20) { /* Better have hit at least 20! */ + cell->sky = cell->rms = 0.0; + return(-1); + } + +/* First pass at sky and quartile */ + cell->sky = int_median(n, median_buf); + cell->rms = 1.33*(cell->sky - median_buf[n/4]); + +/* Clip at 3 sigma */ + for(j=0; median_buf[j] < cell->sky - Q_CLIP*cell->rms && j cell->sky + Q_CLIP*cell->rms && k>n/2; k--); + cell->sky = median_buf[(j+k)/2]; + cell->rms = 1.33 * (cell->sky - median_buf[(3*j+k)/4]); + cell->sky -= cell->bias; + +/* Create mask */ + thresh = cell->bias + cell->sky + MASK_CLIP*cell->rms; + cell->nsat = 0; + for(j=0; jmask[i+NX*j] = 1; + if(cell->data[i+NX*j]+bzero > thresh) cell->mask[i+NX*j] = 0; + if(cell->data[i+NX*j]+bzero > SAT4SURE) (cell->nsat)++; + } + } + + return(0); +} + +static int int_median(int n, int *key) +{ + if(n == 0) return(0); + qsort_int(n, key); + return((key[n/2]+key[(n-1)/2]) / 2); +} + +/* Determine the relative streakiness of all cells in this row */ +/* + * Compare each cell's pixels in the difference which are greater + * than MINSTREAK with the average using pixels surviving the + * intersection of all the masks + */ +#define MINSTREAK 50 +#define EDGE 3 +static int streak_ratio(int sx, int nx, int ny, int NX, CELL otarow[]) +{ + int i, j, k, nalive=0, ncontrib; + double diff; + + for(k=0; k 0) { + nalive++; + } else { + otarow[k].ampl = 0.0; + } + } + +/* Loop over all the pixels */ + ncontrib = 0; + for(j=1+EDGE; j MINSTREAK)) { + for(k=0; k 0) { + k = int_median(n, medimg) + Abuf[j-1] * (i-nx*sx); + sy += k; + s1 += i - (sx+nx); + } + } + Abuf[j] = 0.0; + if(s1 != 0) Abuf[j] = sy / s1; + for(k=0, n=1; k 0) { + k = int_median(n, medimg); + if(i < sx+nx/3) { + s1 += k + Abuf[j-1]; + n1++; + } else { + s2 += k + Bbuf[j-1]; + n2++; + } + } + } + Abuf[j] = Bbuf[j] = 0.0; + if(n1 > 0) Abuf[j] = s1/n1; + if(n2 > 0) Bbuf[j] = s2/n2; + for(k=0, n=1; k 0) { + k = int_median(n, medimg); + sy += k; + s1 += i - (sx+nx); + } + } + if(s1 != 0) A = sy / s1; + +/* Fit a full line z = B*(i-nx-sx)+C to the left third */ + s0 = s1 = s2 = sm = B = C = 0.0; + npt = 0; + for(i=sx+EDGE; i 0) { + k = int_median(n, medimg); + s1 += k; + s0 += (i-sx-nx); + s2 += (i-sx-nx)*(i-sx-nx); + sm += (i-sx-nx)*k; + npt++; + } + } + if(npt > 0) { + B = (npt*sm-s0*s1) / (npt*s2-s0*s0); + C = (s2*s1 - s0*sm) / (npt*s2-s0*s0); + } + + if(A != B) { + i0 = C / (A-B) + sx+nx; + } else { + i0 = 0; + } + +#ifdef DEBUG +// if(otarow[3].mefxt == 4) printf("%4d %4d %9.3f %9.3f %9.3f\n", +// j, i0, A, B, C); +#endif + +/* Correct all the cells */ + for(k=0; k -dbname gpc1 -recipe PPIMAGE CHIP +# +# and then searching the output chip file's header to see what the +# DETREND.MASK, DETREND.DARK, and DETREND.FLAT keywords request. These need +# to be truncated to give the keyword to search for (for example, +# DETREND.MASK = 'GPC1.MASK.20090219.XY10.fits' => GPC1.MASK.20090219 +############################################################################## + +############################################################################## +# The options specify the keywords for each of the detrend steps we care about +############################################################################## +use Getopt::Std; +getopt('d:m:f:h',\%opt); + +if (exists($opt{h})) { + print STDERR "Usage: fetch_detrend.pl [-h] [-f ] [-d ] [-m ]\n"; + exit(1); +} + +############################################################################## +# Set up the database handle, and connect to "gpc1". ######################### +############################################################################## +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ipp004'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + + +############################################################################## +# We have three detrend stages that we need images for, but they can all be +# treated in nearly identical ways. For ease of code maintainance, I've put +# all of them into a single loop. +# +# To make use of that loop, define the stages, and the associated flags. +# +# I apologize for using plural words as arrays, and their singular as an +# element. Some people find it confusing, but it's more intuitive to me. +############################################################################## + +@detrend_stages = ('DARK','MASK','FLAT'); +@flags = ('d','m','f'); + +for ($i = 0; $i <= $#detrend_stages; $i++) { + $detrend_stage = $detrend_stages[$i]; + $flag = $flags[$i]; + + # Check to see that we have a keyword to search for this stage + if (exists($opt{${flag}})) { + # Make the output directory if it doesn't exist. + unless (-d "./${detrend_stage}") { + system("mkdir ./${detrend_stage}"); + } + + print STDERR "$flag -?-> $opt{$flag} -> ${detrend_stage}\n"; + + # The "final" detrends can either live in the detRegisteredImfile + # table or in the detNormalizedImfile table. We search one, and if + # we found no results, we try the other one. I'd like a more elegant + # solution, but this works. + $sth = "SELECT uri FROM detRegisteredImfile WHERE " . + "uri LIKE '%$opt{$flag}%'"; + $ref = $db->selectall_arrayref( $sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + if ($#{ $ref } == -1) { + # This is where we're searching the other one. + $sth = "SELECT uri FROM detNormalizedImfile WHERE " . + "uri LIKE '%$opt{$flag}%'"; + $ref = $db->selectall_arrayref( $sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + # If we still didn't find anything, let's just panic and die, + # because there's clearly some problem somewhere. + if ($#{ $ref } == -1) { + die "Could not find that detrend: $flag : $opt{$flag}\n"; + } + } + + # Let the user know we've done something, by counting our matches. + print STDERR + "$detrend_stage found $#{ $ref } matches for %$opt{$flag}\n"; + + # Loop over each match, and copy the real image file to a directory + # matching the stage of detrending it will be used for. This is + # basically the same as what fetch_rawExp.pl does. There should be + # ~60 of these as well, as you should get one for each detector chip. + foreach $row_ref (@{ $ref }) { + $uri = shift( @{ $row_ref }); + chomp($real_file = + `neb-locate --server http://ipp004/nebulous/ --path $uri`); + print STDERR "$uri -> ./${detrend_stage}/${real_file}\n"; + system("cp $real_file ./${detrend_stage}/"); + } + } # Finish searching for this stage. + else { + # Not all stages need to be done at once, or even at all (since the + # flats will probably change with filter, but the mask and dark do not + # necessarily need to change), but it's always good to tell the user + # that you're at least paying attention to them. + print STDERR "No keyword for $detrend_stage, skipping.\n"; + } +} # Continue to next detrend stage, and then close down. + +$db->disconnect(); Index: branches/eam_branches/20090522/gpc1_test_suite/fetch_rawExp.pl =================================================================== --- branches/eam_branches/20090522/gpc1_test_suite/fetch_rawExp.pl (revision 24557) +++ branches/eam_branches/20090522/gpc1_test_suite/fetch_rawExp.pl (revision 24557) @@ -0,0 +1,123 @@ +#!/usr/bin/perl -w + +############################################################################## +# Script to find the raw .fits images for a given exposure name (like those of +# the form "gTMJDgEXPNo" with TMJD being the truncated modified Julian date, +# and EXPN being the exposure number). Written to solve two problems. First, +# I had no idea how the database was set up, and this was a good way to learn +# that. Second, I needed a way to copy some images to a sandbox to play with +# them safely. +############################################################################## + +############################################################################## +# Set up the database handle, and connect to "gpc1". ######################### +############################################################################## +use DBI; +use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock'; +$dbname = 'gpc1'; +$dbserver = 'ipp004'; +$dbuser = 'ipp'; +$dbpass = 'ipp'; +$db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . + "mysql_socket=" . DB_SOCKET(), + ${dbuser},${dbpass}, + { RaiseError => 1, AutoCommit => 1} + ) or die "Unable to connect to database $DBI::errstr\n"; + + +############################################################################## +# Loop over the input exposure names, and find them in the database.########## +############################################################################## + +# Save the information we find out about the images to a list file, because +# maybe we'll care about that in the future. You can never tell. +open(LIST,">>./rawExp.list") || die "Could not write my list file.\n"; + +while (<>) { + chomp; + $x_name = $_; + + # Grab the same information as the ippMonitor webpage + $sth = "SELECT exp_id,exp_name,dateobs,ra,decl,filter,exp_time,airmass, " . + "bg,bg_stdev,bg_mean_stdev,comment from rawExp WHERE " . + "exp_name like '%$x_name%' "; + $data_ref = $db->selectall_arrayref( $sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + # Store that ippMonitor information in the list file. This adds a quick + # header so that the list is self documenting. + print LIST "#$x_name\n"; + print LIST "#exp_id\texp_name\tdate/time\tRA\tDEC\tfilter\texp_time\t" . + "airmass\tbkg\tsigbkg\tbg_mean_stdev\tcomment\n"; + print STDERR "Working on $x_name\n"; + + # We should only receive one row hit from a fully specified exp_name + # (gTMJDgEXPNo), but the query should match just as well if you specify + # only the date (gTMJDg), so you may possibly receive multiple rows. For + # such an example, you'll receive N rows for the N exposures taken on + # that day. + foreach $row_ref (@{ $data_ref }) { + $exp_id = ${ $row_ref}[0]; + + # Write out the ippMonitor data to the list file. + $list_out = join "\t", @{ $row_ref }; + print LIST "$list_out\n"; + + # Make a directory to hold the .fits images. These are named by the + # exposure id (rawExp.exp_id) to ensure they don't overlap. The list + # file allows these ids to be referenced to the exposure name. + system("mkdir $exp_id"); + print STDERR "\tID: $exp_id\n"; + + # Each detector has information in the database as well. This is + # stored in yet another list, to ensure no loss of information + # compared to ippMonitor (this is the stuff that shows up when you + # click on an exp_id). This is also where we find out the uri + + $c_sth = "SELECT exp_name,class_id,bg,bg_stdev,bg_mean_stdev, " . + "uri FROM rawImfile WHERE " . + "exp_id LIKE '$exp_id'"; # I'm not sure why I used LIKE here. + $chip_ref = $db->selectall_arrayref( $c_sth ) || + die "Unable to execute SQL: $DBI::errstr\n"; + + # Write header for rawImfile.list. + open(CHIPLIST,">>./${exp_id}/rawImfile.list") || + die "Could not write my chip list file: $exp_id.\n"; + print CHIPLIST "#$exp_id\n"; + print CHIPLIST "#exp_name.class\tbkg\tsigbkg\tbg_mean_stdev\turi\n"; + + # This should yield something ~60 rows, which are the individual + # detector chips. Each one is identified, a verbose statement sent + # to the terminal, the rawImfile.list updated, and then the uri + # converted to a real file. + foreach $chip_row_ref (@{ $chip_ref}) { + $c_exp_name = ${ $chip_row_ref }[0]; + $c_class_id = ${ $chip_row_ref }[1]; + $c_uri = ${ $chip_row_ref }[-1]; + print STDERR "\t\t${c_exp_name}.${c_class_id} -> $c_uri\n"; + + $chiplist_out = join "\t", @{ $chip_row_ref }; + $chiplist_out =~ s/\t/./; + print CHIPLIST "$chiplist_out\n"; + + # Use neb-locate to find the real image file, and copy it to the + # data directory set up above. + chomp($real_file = + `neb-locate --server http://ipp004/nebulous/ --path $c_uri`); + system("cp $real_file ./${exp_id}/"); + + } + close(CHIPLIST); + # This closes out this exp_id... + } + # and this closes out the exp_name matches. +} +close(LIST); +# and now we're all done with everything we set out to do. If there haven't +# been any errors, there should be (N > 0) directories named after the exp_id +# they contain the data for. There should be a master output list with the +# database parameters for the exposures, as well as exposure output lists +# with the database parameters for the raw images. Most importantly, though +# the individual .fits images are now copied out of the nebulous system. + +$db->disconnect(); Index: branches/eam_branches/20090522/gpc1_test_suite/gpc1_test.auto =================================================================== --- branches/eam_branches/20090522/gpc1_test_suite/gpc1_test.auto (revision 24557) +++ branches/eam_branches/20090522/gpc1_test_suite/gpc1_test.auto (revision 24557) @@ -0,0 +1,16 @@ +automate MULTI + +automate METADATA + name STR CHIP + regular STR "chiptool -updaterun -set_label @DBNAME@ -dbname @DBNAME@ -inst GPC1" +END + +automate METADATA + name STR STACK + regular STR "stacktool -definebyquery -all -label @DBNAME@ -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@" +END + +#automate METADATA +# name STR DIFF +# regular STR "difftool -definewarpstack -label @DBNAME@ -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@" +#END Index: branches/eam_branches/20090522/gpc1_test_suite/gpc1_test.pro =================================================================== --- branches/eam_branches/20090522/gpc1_test_suite/gpc1_test.pro (revision 24557) +++ branches/eam_branches/20090522/gpc1_test_suite/gpc1_test.pro (revision 24557) @@ -0,0 +1,55 @@ +# gpc1_test.pro : automated test to combine the 2009/06/02 MD07 images +# since converted into a general way to push real GPC1 data +# through the IPP using pantasks. +# Based in no small part from the simtest.pro module + + +macro gpc1_test + if ($0 != 5) + echo "USAGE: gpc1_test " + echo " init = run : only run the analysis tasks" + echo " inject : recreate the database and reinject the images" + echo " new : restart from scratch" + break + end + + $dbname = $1 + $hostname = $2 + $datadir = $3 + $init = $4 + + if (("$init" != "run") && ("$init" != "inject") && ("$init" != "new")) + echo "USAGE: gpc1_test " + echo " init = run : only run the analysis tasks" + echo " inject : recreate the database and reinject the images" + echo " new : restart from scratch" + break + end + + if (("$init" == "new") || ("$init" == "inject")) + exec pxadmin -dbname $dbname -delete + exec pxadmin -dbname $dbname -create + +# exec skycells -mode LOCAL -scale 0.2 -nx 12 -ny 12 -size 4 4 -fix-ns -center 213.146 53.417 -D CATDIR /data/ipp022.0/md07_test/TESS + exec /home/panstarrs/watersc1/bin/mk_project_database.pl -d $dbname -w $datadir + end + + module pantasks.pro + module automate.pro + + module.tasks + + controller host add $hostname + + add.database $dbname + + automate.load gpc1_test.auto GPC1 $dbname + + detrend.off + + add.label $dbname + + + run +end + Index: branches/eam_branches/20090522/gpc1_test_suite/mk_project_database.pl =================================================================== --- branches/eam_branches/20090522/gpc1_test_suite/mk_project_database.pl (revision 24557) +++ branches/eam_branches/20090522/gpc1_test_suite/mk_project_database.pl (revision 24557) @@ -0,0 +1,252 @@ +#!/usr/bin/perl -w +# +# Script to inject data and detrend images into a database. Originally written +# to make the creation of test datasets from raw exposures "easy." + + +############################################################################### +# Load configuration settings. The defaults are set up for my MD07 test suite, +# but can be over-ridden without much difficulty. +############################################################################### +use Getopt::Std; + +$opt{i} = 'GPC1'; +$opt{t} = 'PS1'; +$opt{d} = 'md07_20090602'; +$opt{w} = '/data/ipp022.0/md07_test'; +$opt{f} = 'i'; +getopts('i:t:d:w:f:',\%opt); + +$inst = $opt{i}; +$telescope = $opt{t}; +$dbname = $opt{d}; +$workdir = $opt{w}; +$filter = $opt{f}; + +print STDERR "SETTINGS:\n"; +print STDERR "\tINSTRUMENT: $inst\n"; +print STDERR "\tTELESCOPE: $telescope\n"; +print STDERR "\tDATABASE: $dbname\n"; +print STDERR "\tWORKING DIR: $workdir\n"; +print STDERR "\tFILTER: $filter\n"; +print STDERR "\n"; + +############################################################################### +# Set up the directories we'll be scanning for data. The detrends are expected +# to all be in individual directories. This doesn't gracefully handle multiple +# filters (which change the FLAT), so for right now, each filter needs to be +# loaded individually. The data directories are named after the exposure ID, +# which makes them easy-to-find digits. +# +# Note that you can easily put multiple exposures into a single directory (for +# both the data and the detrend). We scan later to ensure everything gets +# included. +############################################################################### +print STDERR "DIRECTORIES:\n"; +@detrend_dirs_tmp = ("$workdir/ASTROM","$workdir/DARK", + "$workdir/FLAT","$workdir/MASK"); +@detrend_dirs = (); +foreach $ddir (@detrend_dirs_tmp) { + if (-d $ddir) { + push @detrend_dirs, $ddir; + print STDERR "\tFound detrend directory: $ddir\n"; + } +} +print STDERR "\n"; + +@data_dirs = (); +@files = <$workdir/*>; +foreach $f (@files) { + if (-d $f) { + if ($f =~ /^$workdir\/\d+$/) { + push @data_dirs, $f; + print STDERR "\tFound data directory: $f\n"; + } + } +} +print STDERR "\n"; + +############################################################################### +# Process the detrend images. +############################################################################### +print STDERR "DETRENDS:\n"; +foreach $ddir (@detrend_dirs) { + print STDERR "\tScanning $ddir...\n"; + + %detrend_data = (); + + @detfiles = <$ddir/*.fits>; + push @detfiles, <$ddir/*.asm>; + + # Get the exposure name root and the class id for each file in the + # directory. + $det_type = $ddir; + $det_type =~ s%^.*/(.*)$%$1%; + foreach $f (@detfiles) { + ($exp_name,$class_id) = find_expname_classid($f); + $detrend_data{$exp_name}{$class_id} = "$f"; + } + + foreach $exp_name (keys %detrend_data) { + print STDERR "\t\tAdding $exp_name...\n"; + # Build the necessary dettool command string: + $det_id = ''; + if ($det_type eq 'ASTROM') { + $filelevel = "fpa"; + } + else { + $filelevel = "CHIP"; + } + $cmd = "dettool -dbname $dbname " . + "-register_detrend " . + "-det_type $det_type -filelevel $filelevel " . + "-inst $inst -telescope $telescope"; + if ($det_type eq 'FLAT') { + $cmd .= " -filter $filter"; + } + if ($det_type eq 'ASTROM') { + $cmd .= " -use_begin 2008/01/01"; + } + + # Run dettool registration step, and catch the det_id: + print STDERR "\t\t$cmd\n"; + open(DETTOOL,"$cmd |") || die "Failed to run $cmd\n"; + while () { + chomp; + print "DETTOOL:$_\n"; + $_ =~ s/^\s+//; + if ($_ =~ /^det_id/) { + $det_id = (split /\s+/, $_)[2]; + } + } + close(DETTOOL); + if ($det_id eq '') { # Or not, and shut down everything. + die "Did not recieve a det_id.\n"; + } + + # Load the individual image files: + foreach $class_id (sort (keys %{ $detrend_data{$exp_name} })) { + $cmd = "dettool -dbname $dbname " . + "-register_detrend_imfile -det_id $det_id " . + "-class_id $class_id -uri $detrend_data{$exp_name}{$class_id}"; + + print STDERR "\t\t\t$cmd\n"; + system($cmd); + } + + # Tell dettool that we're done with this exposure. + $cmd = "dettool -dbname $dbname " . + "-updatedetrun -state stop -det_id $det_id"; + + print STDERR "\t\t$cmd\n"; + system($cmd); + + } +} +print STDERR "\n"; + +############################################################################## +# Process the data images, doing essentially the same stuff. +############################################################################## +print STDERR "DATA:\n"; +foreach $ddir (@data_dirs) { + print STDERR "\tScanning $ddir...\n"; + + %image_data = (); + + @imfiles = <$ddir/*.fits>; + + # Get the exposure name root and teh class id for each file in the + # directory. + foreach $f (@imfiles) { + ($exp_name,$class_id) = find_expname_classid($f); + $image_data{$exp_name}{$class_id} = "$f"; + } + + foreach $exp_name (keys %image_data) { + print STDERR "\t\tAdding $exp_name...\n"; + # Build the necessary pxinject command string: + $exp_id = ''; + $cmd = "pxinject -dbname $dbname " . + "-newExp -tmp_exp_name $exp_name -workdir $workdir " . + "-tmp_inst $inst -tmp_telescope $telescope -tess_id $workdir/TESS"; + + # Run pxinject registration step, and catch the exp_id: + print STDERR "\t\t$cmd\n"; + open(PXINJECT,"$cmd |") || die "Failed to run $cmd\n"; + while () { + chomp; + $_ =~ s/^\s+//; + if ($_ =~ /^exp_id/) { + $exp_id = (split /\s+/, $_)[2]; + } + } + close(PXINJECT); + if ($exp_id eq '') { # Or not, and shut down everything. + die "Did not receive an exp_id.\n"; + } + + # Load the individual image files: + foreach $class_id (sort (keys %{ $image_data{$exp_name} } )) { + $cmd = "pxinject -dbname $dbname " . + "-newImfile -exp_id $exp_id " . + "-tmp_class_id $class_id " . + "-uri $image_data{$exp_name}{$class_id}"; + + print STDERR "\t\t\t$cmd\n"; + system($cmd); + } + + # Tell pxinject that we're done with this exposure. + $cmd = "pxinject -dbname $dbname " . + "-updatenewExp -state run -exp_id $exp_id"; + + print STDERR "\t\t$cmd\n"; + system($cmd); + } +} +print STDERR "\n"; + + +############################################################################## +# End of script! +############################################################################## +print STDERR "DATABASE INJECTED SUCCESSFULLY\n"; + + +############################################################################## +# Subroutine to read a full filename, and extract out the exposure name and +# class id. Basically just a quick regexp replace. +############################################################################## +sub find_expname_classid { + my $f = shift; + + $f =~ s%^.*/(.*)%$1%; + + if ($f =~ /\.asm$/) { + $exp_name = $f; + $class_id = 'fpa'; + } + elsif ($f =~ /\.XY\d\d.fits/) { + $f =~ s/(.*?)\.XY(\d\d).fits/$1 XY$2/; + ($exp_name,$class_id) = split /\s/, $f; + } + elsif ($f =~ /\.ota\d\d.fits/) { + $f =~ s/(.*?)\.ota(\d\d).fits/$1 ota$2/; + ($exp_name,$class_id) = split /\s/, $f; + } + else { + die "Cannot handle file: $f\n"; + } + return($exp_name,$class_id); +} + + + + + + + + + + Index: branches/eam_branches/20090522/ippMonitor/def/diffRun.d =================================================================== --- branches/eam_branches/20090522/ippMonitor/def/diffRun.d (revision 24529) +++ branches/eam_branches/20090522/ippMonitor/def/diffRun.d (revision 24557) @@ -10,5 +10,5 @@ FIELD magicked, 4, %d, Magicked FIELD tess_id, 8, %s, tess ID -FIELD exp_id, 5, %d, exp ID +FIELD bothways, 5, %d, both ways FIELD registered, 19, %T, Time registered FIELD reduction, 10, %s, reduction Index: branches/eam_branches/20090522/ippMonitor/raw/getimage.php =================================================================== --- branches/eam_branches/20090522/ippMonitor/raw/getimage.php (revision 24529) +++ branches/eam_branches/20090522/ippMonitor/raw/getimage.php (revision 24557) @@ -26,4 +26,5 @@ echo "path: $PATH
"; echo "perl: $PERLLIB
"; + echo "bindir: $BINDIR
"; } Index: branches/eam_branches/20090522/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- branches/eam_branches/20090522/ippMonitor/raw/ipp.imfiles.dat (revision 24529) +++ branches/eam_branches/20090522/ippMonitor/raw/ipp.imfiles.dat (revision 24557) @@ -33,5 +33,5 @@ menulink | menuselect | link | detNormalizedStatImfile | detNormalizedStatImfile.php menulink | menuselect | link | detResidImfile | detResidImfile.php -menulink | menuselect | link | masterDetrendImfiles | masterDetrendImfiles.php +menulink | menuselect | link | detRegisteredImfile | detRegisteredImfile.php menulink | menuselect | link | pzPendingImfile | pzPendingImfile.php menulink | menuselect | link | chipPendingImfile | chipPendingImfile.php Index: branches/eam_branches/20090522/ippScripts/Build.PL =================================================================== --- branches/eam_branches/20090522/ippScripts/Build.PL (revision 24529) +++ branches/eam_branches/20090522/ippScripts/Build.PL (revision 24557) @@ -72,4 +72,5 @@ scripts/ipp_serial_stack.pl scripts/ipp_serial_mops.pl + scripts/ipp_serial_mops_warps.pl scripts/ipp_serial_magic.pl scripts/ipp_mops_translate.pl @@ -90,4 +91,5 @@ scripts/receive_setstatus.pl scripts/rcserver_checkstatus.pl + scripts/publish_file.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: branches/eam_branches/20090522/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/camera_exp.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/camera_exp.pl (revision 24557) @@ -94,5 +94,8 @@ my $recipe_addstar = $ipprc->reduction($reduction, 'ADDSTAR'); # Recipe to use -&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe1; +&my_die("Unrecognised ADDSTAR recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_addstar; + +my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use +&my_die("Unrecognised PSASTRO recipe", $cam_id, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro; my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files @@ -242,5 +245,9 @@ # run psastro on the chipObjects, producing fpaObjects my $command; - $command = "$psastro -list $list3Name -masklist $list4Name $outroot"; + $command = "$psastro"; + $command .= " -list $list3Name"; + $command .= " -masklist $list4Name"; + $command .= " $outroot"; + $command .= " -recipe PSASTRO $recipe_psastro"; $command .= " -tracedest $traceDest -log $logDest"; $command .= " -dbname $dbname" if defined $dbname; Index: branches/eam_branches/20090522/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/chip_imfile.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/chip_imfile.pl (revision 24557) @@ -20,4 +20,5 @@ use PS::IPP::Metadata::Config; use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); @@ -39,5 +40,5 @@ # Parse the command-line arguments my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose, - $no_update, $no_op, $redirect, $magicked, $deburned ); + $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned ); GetOptions( 'exp_id=s' => \$exp_id, # Exposure identifier @@ -58,4 +59,5 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? ) or pod2usage( 2 ); @@ -144,4 +146,54 @@ $uri =~ s/fits$/burn.fits/; &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri); + } + + # apply the 'tiltystreak' operation, if desired + my $tiltystreakApply = metadataLookupBool($recipeData, 'TILTYSTREAK.APPLY'); + if ($tiltystreakApply) { + + my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS'); + my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id"); + if ($tiltystreakClassApply) { + + my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id"); + if ($tiltystreakClassOptions eq "none") { + $tiltystreakClassOptions = ""; + } + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf); + + # XXX make these optional (must be built by psbuild as well...) + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + # create an temporary output file: + my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps ); + + print "uri: $uri\n"; + + # get the UNIX version of the (possible) neb: or path: filename + my $uriReal = $ipprc->file_resolve( $uri ); + + print "uriReal: $uriReal\n"; + + # unpack the data (is a NOP if not compressed) + $command = "$funpack -S $uriReal > $tempName"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + # run tiltystreak + $command = "$tiltystreak $tempName $tiltystreakClassOptions"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + + # supply the output file as the new input file + $uri = $tempName; + } } @@ -201,5 +253,21 @@ } chomp $cmdflags; - ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + if (($camera ne "GPC1") or ($class_id ne "XY27") or ($exp_id < 53171)) { + ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag + } else { + # hack to kick ota 27 out of subsequent analysis by setting quality flag to a non-zero value + my ($before, $after) = split " -quality ", $cmdflags; + + # get the current value + ($quality) = $after =~ /^(\d+)/; + $after = substr($after, length($quality)); + + # replace it if it is zero + $quality = 42 if !$quality; + + # rebuild the cmdflags + $cmdflags = $before . " -quality $quality"; + $cmdflags .= " $after" if $after; + } } @@ -265,5 +333,5 @@ $command = "$chiptool -addprocessedimfile"; $command .= " -exp_id $exp_id"; - $command .= " -uri $outputImage"; + $command .= " -uri $outputImage" if defined $outputImage; $command .= " -path_base $outroot"; $command .= " -hostname $host" if defined $host; Index: branches/eam_branches/20090522/ippScripts/scripts/dist_advancerun.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/dist_advancerun.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/dist_advancerun.pl (revision 24557) @@ -113,5 +113,5 @@ { my $command = "$tool_cmd -exportrun -outfile $dbinfo_file"; - $command .= " -clean" if defined $clean; + $command .= " -clean" if ((defined $clean) and ($stage ne "raw")); $command .= " -dbname $dbname" if defined $dbname; Index: branches/eam_branches/20090522/ippScripts/scripts/dist_component.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/dist_component.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/dist_component.pl (revision 24557) @@ -189,9 +189,9 @@ # save the if ($image_type eq 'image') { - $image = $path + $image = $file_name; } elsif ($image_type eq 'mask') { - $mask = $path; + $mask = $file_name;; } elsif ($image_type eq 'variance') { - $variance = $path; + $variance = $file_name;; } else { &my_die("invalid image type found: $image_type", $dist_id, $component, Index: branches/eam_branches/20090522/ippScripts/scripts/ds9_cmf_regions.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/ds9_cmf_regions.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/ds9_cmf_regions.pl (revision 24557) @@ -23,4 +23,5 @@ $flag_colour, # Flagged source region colour $flag, # Flags + $mag_radius, # Magnitude scaling for radius? $radius, # Radius for circle $save_temps @@ -31,5 +32,11 @@ $flag_colour = "red"; $radius = 5; -$flag = 0x3888; +$flag = (8 | # FAIL + 1024 | # BADPSF + 2048 | # DEFECT + 4096 | # SATURATED + 8192 | # CR_LIMIT + 65536 # SKY_FAILURE + ); GetOptions( @@ -40,4 +47,5 @@ 'flag-colour=s' => \$flag_colour, 'flag=o' => \$flag, + 'mag-radius' => \$mag_radius, 'radius=f' => \$radius, 'save-temps' => \$save_temps, # Save temporary files? @@ -57,18 +65,36 @@ $fits->get_num_rows($numRows, $status) and check_fitsio($status); -my ($xCol, $yCol, $flagCol); # Column numbers for x,y, flag +my ($xCol, $yCol, $flagCol, $magCol); # Column numbers for x,y, flag, mag $fits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status); $fits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status); $fits->get_colnum(0, 'FLAGS', $flagCol, $status) and check_fitsio($status); +$fits->get_colnum(0, 'PSF_INST_MAG', $magCol, $status) and check_fitsio($status); -my ($x, $y, $flags); # Coordinates and flags read from table +my ($x, $y, $flags, $mag); # Coordinates, flags, magnitude read from table $fits->read_col(TFLOAT, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status); $fits->read_col(TFLOAT, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status); $fits->read_col(TINT, $flagCol, 1, 1, $numRows, 0, $flags, undef, $status) and check_fitsio($status); +$fits->read_col(TFLOAT, $magCol, 1, 1, $numRows, 0, $mag, undef, $status) and check_fitsio($status); $fits->close_file($status); + +my ($mag_min, $mag_max); # Minimum magnitude +if ($mag_radius) { + foreach my $m (@$mag) { + next unless defined $m and $m < "inf" and $m > "-inf"; + unless (defined $mag_min and defined $mag_max) { + $mag_min = $m; + $mag_max = $m; + next; + } + $mag_min = $m if $m < $mag_min; + $mag_max = $m if $m > $mag_max; + } +} + my ($coordFile, $coordName) = tempfile( "/tmp/ds9_cmf_regions.XXXX", UNLINK => !$save_temps ); my $numGood = 0; # Number of good sources my $numBad = 0; # Number of bad sources +my $radius_scale = $radius / ($mag_min - $mag_max + 1.0); # Scaling for radius for (my $i = 0; $i < $numRows; $i++) { my $col; # Colour to use @@ -80,5 +106,8 @@ $col = $colour; } - print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$radius) \# color = $col\n"; + my $r = $radius; # Radius of circle + next if defined $mag_radius and (not defined $$mag[$i] or $$mag[$i] == "inf" or $$mag[$i] == "-inf"); + $r -= ($$mag[$i] - $mag_min) * $radius_scale if defined $mag_radius; + print $coordFile "image; circle(" . ($$x[$i] + 1) . ',' . ($$y[$i] + 1) . ",$r) \# color = $col\n"; } close $coordFile; Index: branches/eam_branches/20090522/ippScripts/scripts/ipp_serial_mops.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/ipp_serial_mops.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/ipp_serial_mops.pl (revision 24557) @@ -19,5 +19,5 @@ # Look for programs we need my $missing_tools; -my $mops = can_run('ipp_mops_translate.pl') or (warn "Can't find ipp_mops_translate.pl" and $missing_tools = 1); +my $ppmops = can_run('ppMops') or (warn "Can't find ppMops" and $missing_tools = 1); my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); die "Can't find required tools.\n" if $missing_tools; @@ -72,12 +72,14 @@ die "Unable to connect to database: $DBI::errstr"; -my $where_label = defined $label ? "AND label = '$label'" : ""; # WHERE for label +my $where_label = defined $label ? "AND diffRun.label = '$label'" : ""; # WHERE for label my $sql = " -- Get a list of exposures on which magic may be performed SELECT - exp_id, + rawExp.exp_id, MAX(diffWarps.diff_id) AS diff_id, - -- The following trick pulls out the 'inverse' value for the maximum diff_id + -- The following trick pulls out the appropriate values for the maximum diff_id + SUBSTRING_INDEX(GROUP_CONCAT(camProcessedTemplate.zpt_obs ORDER BY diffWarps.diff_id), ',', 1) AS zpt_obs, + SUBSTRING_INDEX(GROUP_CONCAT(rawTemplate.exp_time ORDER BY diffWarps.diff_id), ',', 1) AS exp_time, CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse FROM ( @@ -86,4 +88,5 @@ diffRun.diff_id, warp1 AS warp_id, + warp1 AS template_warp, 0 AS inverse FROM diffRun @@ -98,4 +101,5 @@ diffRun.diff_id, warp2 AS warp_id, + warp1 AS template_warp, 1 AS inverse FROM diffRun @@ -112,4 +116,10 @@ JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) +JOIN warpRun AS warpTemplate ON warpTemplate.warp_id = diffWarps.template_warp +JOIN fakeRun AS fakeTemplate ON fakeTemplate.fake_id = warpTemplate.fake_id +JOIN camRun AS camTemplate ON camTemplate.cam_id = fakeTemplate.cam_id +JOIN camProcessedExp AS camProcessedTemplate ON camProcessedTemplate.cam_id = camTemplate.cam_id +JOIN chipRun AS chipTemplate ON chipTemplate.chip_id = camTemplate.chip_id +JOIN rawExp AS rawTemplate ON rawTemplate.exp_id = chipTemplate.exp_id WHERE rawExp.camera = '$camera' GROUP BY exp_id;"; @@ -125,6 +135,11 @@ foreach my $diff ( @$diffs ) { my $exp_id = $diff->{exp_id}; + my $zp = $diff->{zpt_obs}; + my $exp_time = $diff->{exp_time}; my $diff_id = $diff->{diff_id}; my $inverse = $diff->{inverse}; + + (carp "Bad ZP or EXPTIME for $exp_id" and next) if not defined $zp or not defined $exp_time; + $zp += 2.5 * log($exp_time) / log(10); my $sql = "SELECT * FROM diffSkyfile WHERE diff_id = $diff_id AND fault = 0 AND quality = 0;"; @@ -146,5 +161,5 @@ unless ($no_op) { $ipprc->file_prepare($output); - my $command = "$mops --input $input --extname SkyChip.psf --output $output"; + my $command = "$ppmops $input $zp $output"; my $success = run( command => $command, verbose => $verbose ); (carp "Couldn't translate $input\n" and next) unless $success; Index: branches/eam_branches/20090522/ippScripts/scripts/ipp_serial_mops_warps.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/ipp_serial_mops_warps.pl (revision 24557) +++ branches/eam_branches/20090522/ippScripts/scripts/ipp_serial_mops_warps.pl (revision 24557) @@ -0,0 +1,145 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use DBI; + +use IPC::Cmd 0.36 qw( can_run run ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use Data::Dumper; +use File::Temp qw( tempfile ); + +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config qw( caturi ); +use Carp qw( carp ); + +# Look for programs we need +my $missing_tools; +my $ppmops = can_run('ppMops') or (warn "Can't find ppMops" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); +die "Can't find required tools.\n" if $missing_tools; + +my $ipprc = PS::IPP::Config->new; # IPP Configuration + +my ( $dbhost, # Database host + $dbname, # Database name + $dbuser, # Database user + $dbpass, # Database p/w + $camera, # Camera used + $outroot, # Output directory + $fileset, # File set + $label, # Data label to search for + $verbose, # Verbose output? + $no_update, # Don't update state? + $no_op, # Don't do any operations? + $save_temps # Save temporary files? + ); + +GetOptions( + 'dbhost=s' => \$dbhost, + 'dbname=s' => \$dbname, + 'dbuser=s' => \$dbuser, + 'dbpass=s' => \$dbpass, + 'camera=s' => \$camera, + 'outroot=s' => \$outroot, + 'fileset=s' => \$fileset, + 'label=s' => \$label, + 'verbose' => \$verbose, + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'save-temps' => \$save_temps, # Save temporary files? +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbhost --dbname --dbuser --dbpass --camera --outroot --fileset", + -exitval => 3) + unless defined $dbhost + and defined $dbname + and defined $dbuser + and defined $dbpass + and defined $camera + and defined $outroot + and defined $fileset; + +$ipprc->define_camera($camera); + +my $dbsrc = 'DBI:mysql:database=' . $dbname . ';host=' . $dbhost . + ';mysql_socket=/var/run/mysqld/mysqld.sock'; +my $db = DBI->connect($dbsrc, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1 } ) or + die "Unable to connect to database: $DBI::errstr"; + +my $where_label = defined $label ? "AND warpRun.label = '$label'" : ""; # WHERE for label + +my $sql = " +-- Get a list of exposures on which magic may be performed +SELECT + rawExp.exp_id, + MAX(warpRun.warp_id) AS warp_id, + -- The following trick pulls out the 'zpt_obs' value for the maximum warp_id + SUBSTRING_INDEX(GROUP_CONCAT(camProcessedExp.zpt_obs ORDER BY warpRun.warp_id), ',', 1) AS zpt_obs +FROM warpRun +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +WHERE rawExp.camera = '$camera' + AND warpRun.state = 'full' + $where_label +GROUP BY exp_id;"; + +my $warps = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr"; + +print "Selected " . scalar @$warps . " rows.\n"; + +$ipprc->outroot_prepare( $outroot ); +my $outrootResolved = $ipprc->file_resolve( $outroot ); +my ($dsFile, $dsName) = tempfile( "$outrootResolved.dslist.XXXX", UNLINK => !$save_temps); + +foreach my $warp ( @$warps ) { + my $exp_id = $warp->{exp_id}; + my $zp = $warp->{zpt_obs}; + my $warp_id = $warp->{warp_id}; + + my $sql = "SELECT * FROM warpSkyfile WHERE warp_id = $warp_id AND fault = 0 AND quality = 0;"; + my $skycells = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr"; + + foreach my $skycell ( @$skycells ) { + my $skycell_id = $skycell->{skycell_id}; + my $path_base = $skycell->{path_base}; + + my $input = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $path_base); + (carp "Can't find $input\n" and next) unless $ipprc->file_exists($input); + $input = $ipprc->file_resolve($input); + + my $output = caturi( $outroot, "mops.$exp_id.$skycell_id.$warp_id.fits" ); + $output = $ipprc->file_resolve($output); + + unless ($no_op) { + $ipprc->file_prepare($output); + my $command = "$ppmops $input $zp $output"; + my $success = run( command => $command, verbose => $verbose ); + (carp "Couldn't translate $input\n" and next) unless $success; + } + + # format: filename|filesize|md5sum|filetype| + # note: since we omit filesize and md5sum, dsreg will calculate them + print $dsFile "${output}|||ipp-mops|\n"; + } +} +close $dsFile; +$db->disconnect; + +# Register new files with the data store +unless ($no_update) { + my $command = "$dsreg --add $fileset --product mops_transient_detections --type MOPS_TRANSIENT_DETECTIONS --list $dsName --copy --abspath --dbname DataStore"; + my $success = run( command => $command, verbose => $verbose ); + die "Couldn't register files with data store.\n" unless $success; +} + + +__END__ + Index: branches/eam_branches/20090522/ippScripts/scripts/magic_destreak.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/magic_destreak.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/magic_destreak.pl (revision 24557) @@ -16,5 +16,5 @@ use IPC::Cmd 0.36 qw( can_run run ); use File::Temp qw( tempfile ); -use File::Basename qw( basename ); +use File::Basename qw( basename dirname ); use PS::IPP::Metadata::Config; use PS::IPP::Metadata::List qw( parse_md_list ); @@ -37,5 +37,5 @@ my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base); my ($outroot, $recoveryroot); -my ($replace, $remove, $release); +my ($replace, $release); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -54,5 +54,4 @@ 'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels 'replace=s' => \$replace, # replace the input images with the results. - 'remove=s' => \$remove, # remove the original images when done YIKES! 'release' => \$release, # NAN masked pixels for release 'save-temps' => \$save_temps, # Save temporary files? @@ -96,8 +95,9 @@ my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $dirname = dirname($path_base); my $basename = basename($path_base); - -# -# parse the replace and remove arguments check for errors and set up +my $nebulousInput = inNebulous($dirname); + +# parse replace arguments check for errors and set up # the appropriate paths if (defined($replace) and ($replace eq "T")) { @@ -108,5 +108,5 @@ # default value is "NULL" do not use -if (defined($recoveryroot) and ($recoveryroot = "NULL")) { +if (defined($recoveryroot) and ($recoveryroot eq "NULL")) { $recoveryroot = undef; } @@ -117,16 +117,6 @@ } -if (defined($remove) and ($remove eq "T")) { - $remove = 1; -} else { - $remove = 0; -} - -if ($remove and !$replace) { - &my_die("cannot remove without replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR); -} - # create the output directories if it is not a nebulous path and it doesn't exist -if (index($outroot, "neb://") != 0) { +if (!inNebulous($outroot)) { if (! -e $outroot ) { my $code = system "mkdir -p $outroot"; @@ -134,14 +124,29 @@ $code >> 8) if $code; } +#} elsif ($replace) { + # if input is in nebulous and we've been asked to replace the input with the destreaked + # version the output must be in nebulous as well + # XXX Is this necessary? if ($replace) the outputs are going to the nebulous "directory" of the + # input anyways. Why require that other files (log) be in nebulous } my $backup_path_base; -if (! $remove) { - $backup_path_base = "$outroot/$basename"; +my $tmproot; +if ($replace) { + # in replace mode, we place the output files in the same "directory" as the inputs + # Nebulous requires this for the two inputs to nebSwap which we use + # We prepend the path with SR_ This causes the filenames for instances of the swapped files to + # have SR in them. + $tmproot = $dirname . "/SR_"; + $backup_path_base = $tmproot . "$basename"; +} else { + # note: trailing / is necessary + $tmproot = "$outroot/"; + $backup_path_base = $tmproot . $basename; } my $recovery_path_base; if ($recoveryroot) { - if (index($recoveryroot, "neb://") != 0) { + if (!inNebulous($recoveryroot)) { if (! -e $recoveryroot ) { my $code = system "mkdir -p $recoveryroot"; @@ -150,5 +155,6 @@ } } - $recovery_path_base = "$recoveryroot/$basename"; + # note: trailing / is necessary + $recovery_path_base = "$recoveryroot/$basename/"; } @@ -209,5 +215,5 @@ { - my $command = "$streaksremove -stage $stage -tmproot $outroot -streaks $streaks -image $image"; + my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image"; $command .= " -class_id $class_id" if defined $class_id; @@ -219,5 +225,4 @@ $command .= " -skycelllist $skycell_list" if defined $skycell_list; $command .= " -replace" if $replace; - $command .= " -remove" if $remove; $command .= " -release" if $release; $command .= " -dbname $dbname" if defined $dbname; @@ -272,4 +277,14 @@ } +sub inNebulous +{ + my $path = shift; + + my $scheme = file_scheme($path); + + return $scheme and ($scheme eq "neb"); +} + + sub my_die { Index: branches/eam_branches/20090522/ippScripts/scripts/magic_process.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/magic_process.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/magic_process.pl (revision 24557) @@ -142,9 +142,21 @@ $command .= " --detect --image $image --mask $mask --weight $weight -k $template"; - # set threshold to 4 sigma - $command .= ' -t 4'; - - # add -S + # XXX: get all of these parameters from a recipe + + # set threshold to 4 sigma + $command .= ' -t 2.3'; +# $command .= ' -t 4'; + + # set Minimum percent visible required between pixels clusters in a valid streak [0.0 - 1.0] + # default is 0.2 + $command .= ' -P 0.2'; + + #$command .= ' -t 4' . ' -P 0.25'; + + # add -S "Number of pixels to shift for determining maximum differences in mean images default:8" $command .= ' -S'; + + # add 3 x 3 binning + $command .= ' -b 3'; # create the list of inputs used at this stage. At higher levels the @@ -349,15 +361,6 @@ my $node = shift; - my $image; # Name of template to return - my $path_base; # Base name for name - if (defined $node->{warp_path_base} and $node->{warp_path_base} ne "NULL") { - $path_base = $node->{warp_path_base}; - $image = "PSWARP.OUTPUT"; - } elsif (defined $node->{stack_path_base} and $node->{stack_path_base} ne "NULL") { - $path_base = $node->{stack_path_base}; - $image = "PPSTACK.OUTPUT"; - } else { - return undef; - } + my $path_base = $node->{diff_path_base}; # Base name for name + my $image = $node->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest $image = $ipprc->file_resolve($ipprc->filename($image, $path_base)); Index: branches/eam_branches/20090522/ippScripts/scripts/publish_file.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/publish_file.pl (revision 24557) +++ branches/eam_branches/20090522/ippScripts/scripts/publish_file.pl (revision 24557) @@ -0,0 +1,198 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename ); +use Carp; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $pubtool = can_run('pubtool') or (warn "Can't find pubtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $ppMops = can_run('ppMops') or (warn "Can't find ppMops" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ( $pub_id, $camera, $stage, $stage_id, $format, $product, $workdir ); +my ( $dbname, $verbose, $no_update, $save_temps, $redirect ); + +GetOptions( + 'pub_id=s' => \$pub_id, # Publish identifier + 'camera=s' => \$camera, # Camera name + 'stage=s' => \$stage, # Stage of interest + 'stage_id=s' => \$stage_id, # Stage identifier + 'product=s' => \$product, # Datastore product name + 'workdir=s' => \$workdir, # Working directory + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'save-temps' => \$save_temps, # Save temporary files? + 'redirect-output' => \$redirect, # Redirect output to log file? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --pub_id --camera --stage --stage_id --product --workdir", + -exitval => $PS_EXIT_CONFIG_ERROR) unless + defined $pub_id and + defined $camera and + defined $product and + defined $stage and + defined $stage_id and + defined $workdir; + +my $outroot = "$workdir/$product.$pub_id"; # Output root name + +my $ipprc = PS::IPP::Config->new( $camera ) or + &my_die( "Unable to set up", $pub_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +$ipprc->outroot_prepare( $outroot ); +my $logDest = "$outroot.log"; +$ipprc->redirect_output($logDest) or &my_die( "Unable to redirect output", $pub_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $mdcParser = PS::IPP::Metadata::Config->new; + + +# Retrieve file name of interest +my %files; # Input filenames +my %zp; # Zero points +{ + my $command; # Command to run + + if ($stage eq 'diff') { + $command = "difftool -diffskyfile -diff_id $stage_id"; + } elsif ($stage eq 'camera') { + $command = "camtool -processedexp -cam_id $stage_id"; + } else { + &my_die( "Unrecognised stage: $stage", $pub_id, $PS_EXIT_CONFIG_ERROR ); + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to retrieve filename", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $pub_id, $PS_EXIT_PROG_ERROR); + + foreach my $comp ( @$components ) { + my $path_base = $comp->{path_base}; # Base name for file + next if defined $comp->{quality} and $comp->{quality} > 0; + + (carp "Bad zpt_obs or exp_time for component" and next) if not defined $comp->{zpt_obs} or not defined $comp->{exp_time}; + my $zp = $comp->{zpt_obs} + 2.5 * log($comp->{exp_time}) / log(10); + + if ($stage eq 'diff') { + my $skycell_id = $comp->{skycell_id}; + $files{"$skycell_id.pos"} = $ipprc->filename( "PPSUB.OUTPUT.SOURCES", $path_base ); + $files{"$skycell_id.neg"} = $ipprc->filename( "PPSUB.INVERSE.SOURCES", $path_base ) if + defined $comp->{bothways} and $comp->{bothways}; + $zp{"$skycell_id.pos"} = $zp; + $zp{"$skycell_id.neg"} = $zp if defined $comp->{bothways} and $comp->{bothways}; + } elsif ($stage eq 'camera') { + my $cam_id = $comp->{cam_id}; + $files{$cam_id} = $ipprc->filename( "PSASTRO.OUTPUT", $path_base ); + $zp{$cam_id} = $zp; + } + } +} + +# Prepare for data store input +my ($listFile, $listFileName) = tempfile("/tmp/publish.$pub_id.list.XXXX", UNLINK => !$save_temps ); + +# Process each file +foreach my $comp ( keys %files ) { + my $file = $ipprc->file_resolve( $files{$comp} ) or + &my_die("Unable to resolve file $files{$comp}", $pub_id, $PS_EXIT_SYS_ERROR); + my_die("Unable to find file $file", $pub_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists( $file ); + + my $zp = $zp{$comp}; + if ($product eq "IPP-MOPS") { + my $out = $ipprc->file_resolve( "$outroot.$comp.fits" ); + my $command = "$ppMops $file $zp $out"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to translate $file", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; + &my_die( "Unable to find translated file $out", $pub_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $out ); + $file = $out; + } + + # format: filename|filesize|md5sum|filetype| + # note: since we omit filesize and md5sum, dsreg will calculate them + print $listFile "$file|||$product|$comp|\n"; +} + + +unless ($no_update) { + my $command = "$dsreg --add $stage.$stage_id --copy --abspath --product $product --type $product --list $listFileName"; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to register with data store", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; +} + +unless ($no_update) { + my $command = "$pubtool -add -pub_id $pub_id -path_base $outroot"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to register with data store", $pub_id, $PS_EXIT_SYS_ERROR) unless $success; +} + + + +### Pau. + + +sub my_die +{ + my $msg = shift; # Exit message + my $pub_id = shift; # Publish run identifier + my $fault = shift; # Fault code + + $fault = $PS_EXIT_PROG_ERROR unless defined $fault; + + carp($msg); + if (defined $pub_id and not $no_update) { + my $command = "$pubtool -add"; + $command .= " -pub_id $pub_id"; + $command .= " -path_base $outroot"; + $command .= " -fault $fault"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + } + exit $fault; +} + + +__END__ Index: branches/eam_branches/20090522/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/20090522/ippScripts/scripts/warp_skycell.pl (revision 24529) +++ branches/eam_branches/20090522/ippScripts/scripts/warp_skycell.pl (revision 24557) @@ -36,5 +36,5 @@ } -my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( 'warp_id|i=s' => \$warp_id, # Warp identifier @@ -44,4 +44,5 @@ 'camera|c=s' => \$camera, # Camera name 'dbname|d=s' => \$dbname, # Database name + 'reduction=s' => \$reduction, # Reduction class 'outroot=s' => \$outroot, # Output root name 'threads=s' => \$threads, # Number of threads to use for pswarp @@ -73,4 +74,11 @@ $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; +my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use for ppImage +unless ($recipe_pswarp) { + &my_die("Couldn't find selected reduction for WARP_PSWARP: $reduction\n", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR); +} my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_WARP); @@ -102,5 +110,5 @@ my $astromSource; # The astrometry source { - my $command = "$ppConfigDump -camera $camera -dump-recipe PSWARP -"; + my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -192,4 +200,5 @@ $command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL"; $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID"; + $command .= " -recipe PSWARP $recipe_pswarp"; $command .= " -psf"; # Turn on PSF determination $command .= " -tracedest $traceDest -log $logDest"; Index: branches/eam_branches/20090522/ippTasks/Makefile.am =================================================================== --- branches/eam_branches/20090522/ippTasks/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/ippTasks/Makefile.am (revision 24557) @@ -23,5 +23,6 @@ rcserver.pro \ pstamp.pro \ - receive.pro + receive.pro \ + publish.pro other_files = \ Index: branches/eam_branches/20090522/ippTasks/ipphosts.mhpcc.config =================================================================== --- branches/eam_branches/20090522/ippTasks/ipphosts.mhpcc.config (revision 24529) +++ branches/eam_branches/20090522/ippTasks/ipphosts.mhpcc.config (revision 24557) @@ -3,13 +3,17 @@ ipphosts METADATA camera STR skycell - count S32 8 - sky00 STR ipp006 - sky01 STR ipp007 - sky02 STR ipp008 - sky03 STR ipp009 - sky04 STR ipp010 - sky05 STR ipp011 - sky06 STR ipp020 - sky07 STR ipp021 + count S32 12 + sky00 STR ipp024 + sky01 STR ipp025 + sky02 STR ipp026 + sky03 STR ipp028 + sky04 STR ipp029 + sky05 STR ipp030 + sky06 STR ipp031 + sky07 STR ipp032 + sky08 STR ipp033 + sky09 STR ipp034 + sky10 STR ipp035 + sky11 STR ipp036 END @@ -40,76 +44,76 @@ XY01 STR ipp014 XY02 STR ipp014 - XY03 STR ipp014 - XY04 STR ipp014 + XY03 STR ipp038 + XY04 STR ipp038 XY05 STR ipp023 XY06 STR ipp023 - XY10 STR ipp023 - XY11 STR ipp023 + XY10 STR ipp039 + XY11 STR ipp039 XY12 STR ipp024 XY13 STR ipp024 - XY14 STR ipp024 - XY15 STR ipp024 + XY14 STR ipp040 + XY15 STR ipp040 XY16 STR ipp026 XY17 STR ipp026 - XY20 STR ipp026 - XY21 STR ipp026 + XY20 STR ipp041 + XY21 STR ipp041 - XY22 STR ipp027 - XY23 STR ipp027 - XY24 STR ipp027 - XY25 STR ipp027 + XY22 STR ipp042 + XY23 STR ipp042 + XY24 STR ipp043 + XY25 STR ipp043 XY26 STR ipp028 XY27 STR ipp028 - XY30 STR ipp028 - XY31 STR ipp028 + XY30 STR ipp044 + XY31 STR ipp044 XY32 STR ipp029 XY33 STR ipp029 - XY34 STR ipp029 - XY35 STR ipp029 + XY34 STR ipp045 + XY35 STR ipp045 XY36 STR ipp030 XY37 STR ipp030 - XY40 STR ipp030 - XY41 STR ipp030 + XY40 STR ipp046 + XY41 STR ipp046 XY42 STR ipp031 XY43 STR ipp031 - XY44 STR ipp031 - XY45 STR ipp031 + XY44 STR ipp047 + XY45 STR ipp047 XY46 STR ipp032 XY47 STR ipp032 - XY50 STR ipp032 - XY51 STR ipp032 + XY50 STR ipp048 + XY51 STR ipp048 XY52 STR ipp033 XY53 STR ipp033 - XY54 STR ipp033 - XY55 S0R ipp033 + XY54 STR ipp049 + XY55 STR ipp049 XY56 STR ipp034 XY57 STR ipp034 - XY60 STR ipp034 - XY61 STR ipp034 + XY60 STR ipp050 + XY61 STR ipp050 XY62 STR ipp035 XY63 STR ipp035 - XY64 STR ipp035 - XY65 STR ipp035 + XY64 STR ipp051 + XY65 STR ipp051 XY66 STR ipp036 XY67 STR ipp036 - XY71 STR ipp036 - XY72 STR ipp036 + XY71 STR ipp052 + XY72 STR ipp052 XY73 STR ipp015 XY74 STR ipp015 - XY75 STR ipp015 - XY76 STR ipp015 + XY75 STR ipp053 + XY76 STR ipp053 END @@ -119,75 +123,75 @@ ota01 STR ipp014 ota02 STR ipp014 - ota03 STR ipp014 - ota04 STR ipp014 + ota03 STR ipp038 + ota04 STR ipp038 ota05 STR ipp023 ota06 STR ipp023 - ota10 STR ipp023 - ota11 STR ipp023 + ota10 STR ipp039 + ota11 STR ipp039 ota12 STR ipp024 ota13 STR ipp024 - ota14 STR ipp024 - ota15 STR ipp024 + ota14 STR ipp040 + ota15 STR ipp040 ota16 STR ipp026 ota17 STR ipp026 - ota20 STR ipp026 - ota21 STR ipp026 + ota20 STR ipp041 + ota21 STR ipp041 - ota22 STR ipp027 - ota23 STR ipp027 - ota24 STR ipp027 - ota25 STR ipp027 + ota22 STR ipp042 + ota23 STR ipp042 + ota24 STR ipp043 + ota25 STR ipp043 ota26 STR ipp028 ota27 STR ipp028 - ota30 STR ipp028 - ota31 STR ipp028 + ota30 STR ipp044 + ota31 STR ipp044 ota32 STR ipp029 ota33 STR ipp029 - ota34 STR ipp029 - ota35 STR ipp029 + ota34 STR ipp045 + ota35 STR ipp045 ota36 STR ipp030 ota37 STR ipp030 - ota40 STR ipp030 - ota41 STR ipp030 + ota40 STR ipp046 + ota41 STR ipp046 ota42 STR ipp031 ota43 STR ipp031 - ota44 STR ipp031 - ota45 STR ipp031 + ota44 STR ipp047 + ota45 STR ipp047 ota46 STR ipp032 ota47 STR ipp032 - ota50 STR ipp032 - ota51 STR ipp032 + ota50 STR ipp048 + ota51 STR ipp048 ota52 STR ipp033 ota53 STR ipp033 - ota54 STR ipp033 - ota55 S0R ipp033 + ota54 STR ipp049 + ota55 STR ipp049 ota56 STR ipp034 ota57 STR ipp034 - ota60 STR ipp034 - ota61 STR ipp034 + ota60 STR ipp050 + ota61 STR ipp050 ota62 STR ipp035 ota63 STR ipp035 - ota64 STR ipp035 - ota65 STR ipp035 + ota64 STR ipp051 + ota65 STR ipp051 ota66 STR ipp036 ota67 STR ipp036 - ota71 STR ipp036 - ota72 STR ipp036 + ota71 STR ipp052 + ota72 STR ipp052 ota73 STR ipp015 ota74 STR ipp015 - ota75 STR ipp015 - ota76 STR ipp015 + ota75 STR ipp053 + ota76 STR ipp053 END Index: branches/eam_branches/20090522/ippTasks/magic.pro =================================================================== --- branches/eam_branches/20090522/ippTasks/magic.pro (revision 24529) +++ branches/eam_branches/20090522/ippTasks/magic.pro (revision 24557) @@ -278,7 +278,18 @@ book getword magicToProcess $pageName dbname -var DBNAME -# set.host.for.camera $CAMERA $MAGIC_ID -# set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR - host anyhost +# XXX: we need new macros that select the host and workdir based on the NODE +# we also need to make the magic scripts able to deal with the workdir being spread +# around. That should not be too hard +# For now select nodes by skycell + + substr $NODE 0 3 NODE_HEAD + if ("$NODE_HEAD" == "sky") + set.host.for.skycell $NODE + else + host anyhost + end + +# currently DetectStreaks expects that the workdir is not in nebulous +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR $WORKDIR = $WORKDIR_TEMPLATE @@ -410,5 +421,4 @@ book getword magicToDS $pageName recoveryroot -var RECROOT book getword magicToDS $pageName re_place -var REPLACE - book getword magicToDS $pageName remove -var REMOVE book getword magicToDS $pageName dbname -var DBNAME @@ -417,7 +427,7 @@ host anyhost - # TODO: do not add recoveryroot, replace, or remove if they are null or zero - - $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --remove $REMOVE + # TODO: do not add recoveryroot or replace if they are null or zero + + $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE if ("$INVERSE" == "T") # Inverse subtraction Index: branches/eam_branches/20090522/ippTasks/publish.pro =================================================================== --- branches/eam_branches/20090522/ippTasks/publish.pro (revision 24557) +++ branches/eam_branches/20090522/ippTasks/publish.pro (revision 24557) @@ -0,0 +1,201 @@ +## publish.pro: -*- sh -*- + +# test for required global variables +check.globals + +book init publishRun + +macro publish.status + book listbook publishRun +end + +macro publish.reset + book init publishRun +end + +macro publish.on + task publish.trigger + active true + end + task publish.load + active true + end + task publish.run + active true + end +end + +macro publish.off + task publish.trigger + active false + end + task publish.load + active false + end + task publish.run + active false + end +end + + +# this variable will cycle through the known database names +$publish_trigger_DB = 0 +$publish_load_DB = 0 + +task publish.trigger + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/publish.trigger.log + + task.exec + $run = pubtool -definerun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$publish_trigger_DB + $run = $run -dbname $DB:$publish_trigger_DB + $publish_trigger_DB ++ + if ($publish_trigger_DB >= $DB:n) set publish_trigger_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task publish.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/publish.load.log + + task.exec + $run = pubtool -pending + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$publish_load_DB + $run = $run -dbname $DB:$publish_load_DB + $publish_load_DB ++ + if ($publish_load_DB >= $DB:n) set publish_load_DB = 0 + end + add_poll_args run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout publishRun -key pub_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook publishRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup publishRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task publish.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + book npages publishRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + + book getpage publishRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword publishRun $pageName pantaskState RUN + book getword publishRun $pageName pub_id -var PUB_ID + book getword publishRun $pageName camera -var CAMERA + book getword publishRun $pageName workdir -var WORKDIR_TEMPLATE + book getword publishRun $pageName product -var PRODUCT + book getword publishRun $pageName stage -var STAGE + book getword publishRun $pageName stage_id -var STAGE_ID + book getword publishRun $pageName dbname -var DBNAME + + stdout $LOGDIR/publish.run.log + stderr $LOGDIR/publish.run.log + + host anyhost + strsub $WORKDIR_TEMPLATE @HOST@ $default_host -var WORKDIR + + $run = publish_file.pl --pub_id $PUB_ID --camera $CAMERA --workdir $WORKDIR --product $PRODUCT --stage $STAGE --stage_id $STAGE_ID --redirect-output + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit publishRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword publishRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword publishRun $options:0 pantaskState TIMEOUT + end +end Index: branches/eam_branches/20090522/ippTasks/site.mhpcc.pro =================================================================== --- branches/eam_branches/20090522/ippTasks/site.mhpcc.pro (revision 24529) +++ branches/eam_branches/20090522/ippTasks/site.mhpcc.pro (revision 24557) @@ -15,5 +15,4 @@ controller host add ipp024 controller host add ipp026 - controller host add ipp027 controller host add ipp028 controller host add ipp029 @@ -25,5 +24,20 @@ controller host add ipp035 controller host add ipp036 - + controller host add ipp038 + controller host add ipp039 + controller host add ipp040 + controller host add ipp041 + controller host add ipp042 + controller host add ipp043 + controller host add ipp044 + controller host add ipp045 + controller host add ipp046 + controller host add ipp047 + controller host add ipp048 + controller host add ipp049 + controller host add ipp050 + controller host add ipp051 + controller host add ipp052 + controller host add ipp053 end Index: branches/eam_branches/20090522/ippTasks/warp.pro =================================================================== --- branches/eam_branches/20090522/ippTasks/warp.pro (revision 24529) +++ branches/eam_branches/20090522/ippTasks/warp.pro (revision 24557) @@ -297,4 +297,5 @@ # XXX change tess_id to tess_dir when schema is changed book getword warpPendingSkyCell $pageName tess_id -var TESS_DIR + book getword warpPendingSkyCell $pageName reduction -var REDUCTION book getword warpPendingSkyCell $pageName exp_tag -var EXP_TAG book getword warpPendingSkyCell $pageName state -var RUN_STATE @@ -306,5 +307,4 @@ end - # set the host and workdir based on the skycell hash set.host.for.skycell $SKYCELL_ID @@ -318,4 +318,7 @@ $run = warp_skycell.pl --threads @MAX_THREADS@ --warp_id $WARP_ID --warp_skyfile_id $WARP_SKYFILE_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE $MAGICKED_ARG + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end add_standard_args run Index: branches/eam_branches/20090522/ippTools/configure.ac =================================================================== --- branches/eam_branches/20090522/ippTools/configure.ac (revision 24529) +++ branches/eam_branches/20090522/ippTools/configure.ac (revision 24557) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.53]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.54]) PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}" Index: branches/eam_branches/20090522/ippTools/doc/publish_flow.txt =================================================================== --- branches/eam_branches/20090522/ippTools/doc/publish_flow.txt (revision 24557) +++ branches/eam_branches/20090522/ippTools/doc/publish_flow.txt (revision 24557) @@ -0,0 +1,14 @@ +### Setup +pubtool -defineclient -stage diff -comment Comment -product IPP-MOPS -workdir /path/to/somewhere + +### Regularly run by pantasks to generate publishing runs based on data that has been processed +pubtool -definerun + +### Regularly run by pantasks to get list of files to publish +pubtool -pending +--> publish_file.pl --pub_id 12345 --workdir /path/to/somewhere --product IPP-MOPS --stage diff --stage_id 67890 +----> pubtool -add -pub_id 12345 -path_base /path/to/somewherenice -fault 0 + +### Run as required +pubtool -revert + Index: branches/eam_branches/20090522/ippTools/share/Makefile.am =================================================================== --- branches/eam_branches/20090522/ippTools/share/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/Makefile.am (revision 24557) @@ -1,6 +1,5 @@ pxadmin_create_mirror_tables.sql: pxadmin_create_tables.sql -$(RM) pxadmin_create_mirror_tables.sql - $(SED) -n -e '/^ *$$/ d' -e 's/--.*//' -e '/./ H' -e '/;/ !b' -e 'x ; s/\n//g ; /^CREATE TABLE receive/ { b print }' -e 's/AUTO_INCREMENT//g ; s/,\{0,1\} *FOREIGN *KEY *([a-zA-Z][a-zA-Z0-9\-\_, ]*) *REFERENCES *[a-zA-Z]\{1,\}([a-zA-Z][a-zA-Z0-9\-\_, ]*)//g' -e ': print' -e 's/( */(/g ; s/ *)/)/g ; s/ */ /g ; p ; s/.*//g ; x' pxadmin_create_tables.sql > pxadmin_create_mirror_tables.sql - echo '-- This comment line is here to avoid empty query error.' >> pxadmin_create_mirror_tables.sql + ./parse_pxadmin_create_tables.pl pxadmin_create_tables.sql > pxadmin_create_mirror_tables.sql dist_pkgdata_DATA = \ @@ -154,8 +153,10 @@ magictool_rawimfile.sql \ magictool_revertnode.sql \ + magicdstool_completed_runs.sql \ + magicdstool_getrunids.sql \ + magicdstool_getskycells.sql \ magicdstool_todestreak.sql \ - magicdstool_completed_runs.sql \ - magicdstool_getskycells.sql \ - magicdstool_getrunids.sql \ + magicdstool_toremove.sql \ + magicdstool_torestore.sql \ pstamptool_addjob_otherjob.sql \ pstamptool_addjob_stampjob.sql \ @@ -167,4 +168,7 @@ pxadmin_create_mirror_tables.sql \ pxadmin_drop_tables.sql \ + pubtool_definerun.sql \ + pubtool_pending.sql \ + pubtool_revert.sql \ pztool_find_completed_exp.sql \ pztool_pendingimfile.sql \ @@ -221,5 +225,5 @@ warptool_pendingcleanuprun.sql \ warptool_pendingcleanupskyfile.sql \ - warptool_revertwarped_update.sql \ + warptool_revertoverlap.sql \ warptool_revertwarped_delete.sql \ warptool_scmap.sql \ Index: branches/eam_branches/20090522/ippTools/share/camtool_find_processedexp.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/camtool_find_processedexp.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/camtool_find_processedexp.sql (revision 24557) @@ -5,4 +5,5 @@ rawExp.exp_tag, rawExp.exp_name, + rawExp.exp_time, rawExp.camera, rawExp.telescope, Index: branches/eam_branches/20090522/ippTools/share/difftool_skyfile.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/difftool_skyfile.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/difftool_skyfile.sql (revision 24557) @@ -5,4 +5,11 @@ diffRun.state, diffRun.workdir, + diffRun.bothways, + camProcessedExp.zpt_obs, + camProcessedExp.zpt_stdev, + camProcessedExp.zpt_lq, + camProcessedExp.zpt_uq, + rawExp.exp_time, + rawExp.camera, warp1, stack1, @@ -13,7 +20,11 @@ JOIN diffInputSkyfile USING(diff_id, skycell_id) JOIN warpRun +-- NOTE: joining input only! +-- This is so that we can get the correct zero point +-- XXX This needs to be more clever to handle diffs between stacks ON warpRun.warp_id = diffInputSkyfile.warp1 JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) Index: branches/eam_branches/20090522/ippTools/share/disttool_pendingfileset.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/disttool_pendingfileset.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/disttool_pendingfileset.sql (revision 24557) @@ -16,3 +16,6 @@ LEFT JOIN rcDSFileset USING(prod_id, dist_id) WHERE distRun.state = 'full' + AND rcDestination.state = 'enabled' + AND distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' AND rcDSFileset.fs_id IS NULL Index: branches/eam_branches/20090522/ippTools/share/disttool_revertrun_delete.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/disttool_revertrun_delete.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/disttool_revertrun_delete.sql (revision 24557) @@ -2,3 +2,4 @@ USING distComponent, distRun WHERE distComponent.dist_id = distRun.dist_id + AND distRun.state = 'new' AND distComponent.fault != 0 Index: branches/eam_branches/20090522/ippTools/share/disttool_revertrun_update.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/disttool_revertrun_update.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/disttool_revertrun_update.sql (revision 24557) @@ -1,5 +1,4 @@ UPDATE distRun -JOIN distComponent USING(dist_id) -SET distRun.state = 'new', distRun.fault = 0 -WHERE distComponent.fault != 0 - OR distRun.fault != 0 +SET distRun.fault = 0 +WHERE distRun.state = 'new' + AND distRun.fault != 0 Index: branches/eam_branches/20090522/ippTools/share/faketool_pendingimfile.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/faketool_pendingimfile.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/faketool_pendingimfile.sql (revision 24557) @@ -1,48 +1,46 @@ -SELECT DISTINCT * FROM ( - -- the subselect is so where criteria can be specified without knowing - -- which table the field came from - SELECT - fakeRun.*, - chipProcessedImfile.exp_id, - chipProcessedImfile.class_id, - chipProcessedImfile.uri, - chipProcessedImfile.bg, - chipProcessedImfile.bg_stdev, - chipProcessedImfile.bg_mean_stdev, - chipProcessedImfile.fringe_0, - chipProcessedImfile.fringe_1, - chipProcessedImfile.fringe_2, - chipProcessedImfile.ap_resid, - chipProcessedImfile.ap_resid_stdev, - chipProcessedImfile.n_stars, - chipProcessedImfile.n_extended, - chipProcessedImfile.n_cr, - chipProcessedImfile.path_base as chip_path_base, - camProcessedExp.path_base as cam_path_base, - rawExp.exp_name, - rawExp.exp_tag, - rawExp.camera, - rawExp.telescope, - rawExp.filelevel - FROM fakeRun - JOIN camRun USING(cam_id) - JOIN camProcessedExp USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN chipProcessedImfile USING(chip_id) - JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id - LEFT JOIN fakeProcessedImfile - ON fakeRun.fake_id = fakeProcessedImfile.fake_id - AND chipProcessedImfile.class_id = fakeProcessedImfile.class_id - LEFT JOIN fakeMask - ON fakeRun.label = fakeMask.label - WHERE - ((fakeRun.state = 'new' - AND fakeProcessedImfile.fake_id IS NULL - AND fakeProcessedImfile.class_id IS NULL - ) - OR (fakeRun.state = 'update' - AND fakeProcessedImfile.data_state = 'cleaned') - ) - AND fakeMask.label IS NULL - ) as fakePendingImfile +-- the subselect is so where criteria can be specified without knowing +-- which table the field came from +SELECT DISTINCT + fakeRun.*, + chipProcessedImfile.exp_id, + chipProcessedImfile.class_id, + chipProcessedImfile.uri, + chipProcessedImfile.bg, + chipProcessedImfile.bg_stdev, + chipProcessedImfile.bg_mean_stdev, + chipProcessedImfile.fringe_0, + chipProcessedImfile.fringe_1, + chipProcessedImfile.fringe_2, + chipProcessedImfile.ap_resid, + chipProcessedImfile.ap_resid_stdev, + chipProcessedImfile.n_stars, + chipProcessedImfile.n_extended, + chipProcessedImfile.n_cr, + chipProcessedImfile.path_base as chip_path_base, + camProcessedExp.path_base as cam_path_base, + rawExp.exp_name, + rawExp.exp_tag, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel +FROM fakeRun +JOIN camRun USING(cam_id) +JOIN camProcessedExp USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN chipProcessedImfile USING(chip_id) +JOIN rawExp + ON chipProcessedImfile.exp_id = rawExp.exp_id +LEFT JOIN fakeProcessedImfile + ON fakeRun.fake_id = fakeProcessedImfile.fake_id + AND chipProcessedImfile.class_id = fakeProcessedImfile.class_id +LEFT JOIN fakeMask + ON fakeRun.label = fakeMask.label +WHERE + ((fakeRun.state = 'new' + AND fakeProcessedImfile.fake_id IS NULL + AND fakeProcessedImfile.class_id IS NULL + ) + OR (fakeRun.state = 'update' + AND fakeProcessedImfile.data_state = 'cleaned') + ) + AND fakeMask.label IS NULL Index: branches/eam_branches/20090522/ippTools/share/magicdstool_completed_runs.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/magicdstool_completed_runs.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/magicdstool_completed_runs.sql (revision 24557) @@ -32,4 +32,5 @@ magicDSRun.state = 'new' AND magicDSRun.stage = 'chip' + AND chipProcessedImfile.quality = 0 GROUP BY magic_ds_id, Index: branches/eam_branches/20090522/ippTools/share/magicdstool_toremove.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/magicdstool_toremove.sql (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/magicdstool_toremove.sql (revision 24557) @@ -0,0 +1,11 @@ +SELECT + magic_ds_id, + stage, + stage_id, + component, + backup_path_base +FROM magicDSFile +JOIN magicDSRun USING(magic_ds_id) +WHERE magicDSRun.remove + AND magicDSRun.state = 'full' + AND backup_path_base IS NOT NULL Index: branches/eam_branches/20090522/ippTools/share/magicdstool_torestore.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/magicdstool_torestore.sql (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/magicdstool_torestore.sql (revision 24557) @@ -0,0 +1,12 @@ +SELECT + magic_ds_id, + stage, + stage_id + component, + backup_path_base, + recovery_path_base +FROM magicDSFile +JOIN magicDSRun USING(magic_ds_id) +WHERE magicDSRun.state = 'goto_restored' + AND backup_path_base IS NOT NULL + AND recovery_path_base IS NOT NULL Index: branches/eam_branches/20090522/ippTools/share/parse_pxadmin_create_tables.pl =================================================================== --- branches/eam_branches/20090522/ippTools/share/parse_pxadmin_create_tables.pl (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/parse_pxadmin_create_tables.pl (revision 24557) @@ -0,0 +1,34 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +my $ctr=0; +my $old=""; +while (<>) { + s/^\s+//; #remove whitespace from front + s/\s*--.*\n$//; # remove all comments + s/\s+/ /g; # kill the middle whitespace + s/\s*$//; # kill the whitespace at the end of the line +# if ($_ !~ /;$/ ) {chomp $_}; + s/;$/;\n/; # add back in the \n that was removed as whitespace but only for ; + s/,$/, /; # add space at end of , if at end of line + if (m/CREATE\s+TABLE\s+receive/) {$ctr = 1;} + if (m/;/) {$ctr = 0;} + if ($ctr == 0) { + s/AUTO_INCREMENT// ; + if (/FOREIGN\s+KEY/) { + if (!/, $/) { #if FOREIGN KEY line doesn't have , + $old =~ s/,\s*$//; #remove , from previous line + print $old; + $old=""; + } + } else { + print $old; + $old=$_; + } + } else { print $_; + } +} +print $old; +print "-- This comment line is here to avoid empty query error.\n"; Index: branches/eam_branches/20090522/ippTools/share/pubtool_definerun.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/pubtool_definerun.sql (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/pubtool_definerun.sql (revision 24557) @@ -0,0 +1,28 @@ +-- Get runs to publish +SELECT + client_id, + stage_id +FROM ( + -- Get diffs to publish + SELECT + client_id, + diff_id AS stage_id + FROM publishClient + JOIN diffRun + WHERE publishClient.stage = 'diff' + AND diffRun.state = 'full' + -- WHERE hook %s + UNION + -- Get cameras to publish + SELECT + client_id, + cam_id AS stage_id + FROM publishClient + JOIN camRun + WHERE publishClient.stage = 'camera' + AND camRun.state = 'full' + -- WHERE hook %s + ) AS publishToDo +-- Only get stuff that hasn't been published +LEFT JOIN publishRun USING(client_id, stage_id) +WHERE publishRun.client_id IS NULL Index: branches/eam_branches/20090522/ippTools/share/pubtool_pending.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/pubtool_pending.sql (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/pubtool_pending.sql (revision 24557) @@ -0,0 +1,48 @@ +SELECT + publishToDo.* +FROM ( + -- Get the diffs + -- The following is only appropriate for a diff where warp1 is set; otherwise it's more difficult to get the camera name + SELECT DISTINCT + publishRun.pub_id, + publishClient.product, + publishClient.stage, + publishClient.workdir, + diffRun.diff_id AS stage_id, + rawExp.camera + FROM publishRun + JOIN publishClient USING(client_id) + JOIN diffRun + ON diffRun.diff_id = publishRun.stage_id + JOIN diffInputSkyfile USING(diff_id) + -- Need to do something fancy here to get the camera name for a stack + LEFT JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'diff' + AND publishRun.state = 'new' + AND diffRun.state = 'full' + -- WHERE hook %s + UNION + SELECT + publishRun.pub_id, + publishClient.product, + publishClient.stage, + publishClient.workdir, + camRun.cam_id AS stage_id, + rawExp.camera + FROM publishRun + JOIN publishClient USING(client_id) + JOIN camRun + ON camRun.cam_id = publishRun.stage_id + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + WHERE publishClient.stage = 'camera' + AND publishRun.state ='new' + AND camRun.state = 'full' + -- WHERE hook %s +) AS publishToDo +LEFT JOIN publishDone USING(pub_id) +WHERE publishDone.pub_id IS NULL Index: branches/eam_branches/20090522/ippTools/share/pubtool_revert.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/pubtool_revert.sql (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/pubtool_revert.sql (revision 24557) @@ -0,0 +1,5 @@ +DELETE FROM publishDone +USING publishDone, publishRun, publishClient +WHERE publishDone.pub_id = publishRun.pub_id + AND publishRun.client_id = publishClient.client_id + AND publishDone.fault != 0 Index: branches/eam_branches/20090522/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/pxadmin_create_tables.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/pxadmin_create_tables.sql (revision 24557) @@ -163,5 +163,5 @@ fault SMALLINT NOT NULL, epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(exp_id), KEY(exp_name), @@ -236,5 +236,5 @@ epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP, raw_image_id BIGINT AUTO_INCREMENT, - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(exp_id, class_id), KEY(tmp_class_id), @@ -259,5 +259,5 @@ tess_id VARCHAR(64), end_stage VARCHAR(64), - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(chip_id), KEY(chip_id), KEY(exp_id), @@ -794,7 +794,8 @@ dvodb VARCHAR(255), tess_id VARCHAR(64), + reduction VARCHAR(64), end_stage VARCHAR(64), registered DATETIME, - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(warp_id), KEY(warp_id), @@ -849,5 +850,5 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT, - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(warp_id, skycell_id, tess_id), KEY(good_frac), @@ -936,5 +937,5 @@ bothways TINYINT DEFAULT 0, exposure TINYINT DEFAULT 0, - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(diff_id), KEY(diff_id), @@ -995,5 +996,5 @@ quality SMALLINT NOT NULL DEFAULT 0, fault SMALLINT, - magicked TINYINT, + magicked BIGINT, PRIMARY KEY(diff_id, skycell_id), KEY(good_frac), @@ -1092,6 +1093,8 @@ recovery_path_base VARCHAR(255), fault SMALLINT, + data_state VARCHAR(64), PRIMARY KEY(magic_ds_id, component), KEY(fault), + KEY(data_state), FOREIGN KEY(magic_ds_id) REFERENCES magicDSRun(magic_ds_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1368,4 +1371,43 @@ + +-- Tables to support publishing of detections to a Science Client + +-- Clients to which we send stuff +CREATE TABLE publishClient ( + client_id BIGINT AUTO_INCREMENT, -- unique identifier + product VARCHAR(64), -- product name + stage VARCHAR(64) NOT NULL, -- stage of interest (chip, camera, diff, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + comment VARCHAR(255), -- for human memory + PRIMARY KEY(client_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Publishing a set of data (e.g., a specific diffRun) +CREATE TABLE publishRun ( + pub_id BIGINT AUTO_INCREMENT, -- unique identifier + client_id BIGINT NOT NULL, -- link to publishClient + stage_id BIGINT NOT NULL, -- link to various stage tables + label VARCHAR(64), -- label for run + state VARCHAR(64), -- state of run (new, full, etc.) + PRIMARY KEY(pub_id), + KEY(client_id), + KEY(stage_id), + KEY(label), + KEY(state), + FOREIGN KEY(client_id) REFERENCES publishClient(client_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Publishing a file within a set +CREATE TABLE publishDone ( + pub_id BIGINT AUTO_INCREMENT, -- link to publishRun + path_base VARCHAR(255), -- base path of output + fault SMALLINT NOT NULL DEFAULT 0, -- Fault code + PRIMARY KEY(pub_id), + KEY(fault), + FOREIGN KEY(pub_id) REFERENCES publishRun(pub_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + -- This comment line is here to avoid empty query error. -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy. Index: branches/eam_branches/20090522/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/pxadmin_drop_tables.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/pxadmin_drop_tables.sql (revision 24557) @@ -72,4 +72,7 @@ DROP TABLE IF EXISTS receiveFile; DROP TABLE IF EXISTS receiveResult; +DROP TABLE IF EXISTS publishClient; +DROP TABLE IF EXISTS publishRun; +DROP TABLE IF EXISTS publishDone; SET FOREIGN_KEY_CHECKS=1 Index: branches/eam_branches/20090522/ippTools/share/warptool_revertoverlap.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/warptool_revertoverlap.sql (revision 24557) +++ branches/eam_branches/20090522/ippTools/share/warptool_revertoverlap.sql (revision 24557) @@ -0,0 +1,10 @@ +DELETE FROM warpSkyCellMap +USING warpSkyCellMap, warpRun, fakeRun, camRun, chipRun, rawExp +WHERE warpSkyCellMap.warp_id = warpRun.warp_id + AND warpRun.fake_id = fakeRun.fake_id + AND fakeRun.cam_id = camRun.cam_id + AND camRun.chip_id = chipRun.chip_id + AND chipRun.exp_id = rawExp.exp_id + AND warpRun.state = 'new' + AND warpSkyCellMap.fault != 0 + Index: branches/eam_branches/20090522/ippTools/share/warptool_revertwarped_delete.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/warptool_revertwarped_delete.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/warptool_revertwarped_delete.sql (revision 24557) @@ -6,4 +6,5 @@ AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id + AND warpRun.state = 'new' AND warpSkyfile.fault != 0 Index: branches/eam_branches/20090522/ippTools/share/warptool_revertwarped_update.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/warptool_revertwarped_update.sql (revision 24529) +++ (revision ) @@ -1,8 +1,0 @@ -UPDATE warpRun -JOIN warpSkyfile USING(warp_id) -JOIN fakeRun USING(fake_id) -JOIN camRun USING(cam_id) -JOIN chipRun USING(chip_id) -JOIN rawExp USING(exp_id) -SET warpRun.state = 'new' -WHERE warpSkyfile.fault != 0 Index: branches/eam_branches/20090522/ippTools/share/warptool_towarped.sql =================================================================== --- branches/eam_branches/20090522/ippTools/share/warptool_towarped.sql (revision 24529) +++ branches/eam_branches/20090522/ippTools/share/warptool_towarped.sql (revision 24557) @@ -6,4 +6,5 @@ warpRun.fake_id, warpRun.state, + warpRun.reduction, camRun.cam_id, rawExp.camera, Index: branches/eam_branches/20090522/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/20090522/ippTools/src/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/Makefile.am (revision 24557) @@ -20,5 +20,6 @@ stacktool \ warptool \ - receivetool + receivetool \ + pubtool @@ -59,5 +60,6 @@ regtool.h \ stacktool.h \ - warptool.h + warptool.h \ + pubtool.h lib_LTLIBRARIES = libpxtools.la @@ -220,4 +222,10 @@ receivetoolConfig.c +pubtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +pubtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +pubtool_SOURCES = \ + pubtool.c \ + pubtoolConfig.c + clean-local: -rm -f TAGS Index: branches/eam_branches/20090522/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/chiptool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/chiptool.c (revision 24557) @@ -473,5 +473,5 @@ PXOPT_LOOKUP_F32(n_cr, config->args, "-n_cr", false, false); PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // default values @@ -601,5 +601,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); - PXOPT_COPY_S32(config->args, where, "-magicked", "chipRun.magicked", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); if (!psListLength(where->list) && Index: branches/eam_branches/20090522/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/difftool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/difftool.c (revision 24557) @@ -49,5 +49,5 @@ static bool importrunMode(pxConfig *config); -static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked); +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); static bool diffRunComplete(pxConfig *config); @@ -512,5 +512,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); if (!psDBTransaction(config->dbh)) { @@ -739,5 +739,5 @@ -static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked) +static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked) { PS_ASSERT_PTR_NON_NULL(state, false); @@ -749,5 +749,5 @@ } - char *query = "UPDATE diffRun SET state = '%s', magicked = %d WHERE diff_id = %"PRId64; + char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { @@ -803,5 +803,5 @@ false, false, - false // magicked + 0 // magicked ); @@ -1190,5 +1190,5 @@ false, // bothways true, // exposure - false // magicked + 0 // magicked ); @@ -1805,5 +1805,5 @@ psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - bool magicked = psMetadataLookupBool(NULL, row, "magicked"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); // set diffRun.state to 'stop' Index: branches/eam_branches/20090522/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/disttool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/disttool.c (revision 24557) @@ -488,5 +488,9 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "fault", "=="); + + // we need to disambiguate fault so make a copy of the where list before adding fault + psMetadata *whereComponent = psMetadataCopy(NULL, where); + PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); + PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "=="); // It might be useful to be able to query by the parameters of the underlying runs @@ -533,15 +537,4 @@ numUpdated = psDBAffectedRows(config->dbh); - -#ifdef notdef - // don't need this. distRun.state may still be in 'new' state - if (numUpdated < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } -#endif } @@ -560,6 +553,6 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + if (psListLength(whereComponent->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -582,4 +575,5 @@ psFree(where); + psFree(whereComponent); if (!psDBCommit(config->dbh)) { @@ -598,5 +592,5 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -719,5 +713,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -863,5 +857,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); Index: branches/eam_branches/20090522/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/disttoolConfig.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/disttoolConfig.c (revision 24557) @@ -98,5 +98,5 @@ psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); - psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", 0, "limit results to label", NULL); + psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -121,5 +121,5 @@ psMetadata *toadvanceArgs = psMetadataAlloc(); psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", 0, "limit updates to label", NULL); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit updates to label", NULL); psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -128,5 +128,5 @@ psMetadata *pendingfilesetArgs = psMetadataAlloc(); psMetadataAddS64(pendingfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", 0, "limit results to label", NULL); + psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -145,5 +145,5 @@ psMetadataAddU64(pendingdestArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingdestArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddStr(pendingdestArgs, PS_LIST_TAIL, "-label", 0, "limit results to label", NULL); + psMetadataAddStr(pendingdestArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); // -revertfileset Index: branches/eam_branches/20090522/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/faketool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/faketool.c (revision 24557) @@ -235,8 +235,8 @@ bool status; char *end_stage = psMetadataLookupStr(&status, md, "end_stage"); - if (end_stage && strcasecmp(end_stage, "warp")) continue; + if (end_stage && strcasecmp(end_stage, "warp")) continue; char *raw_tess_id = psMetadataLookupStr(&status, md, "tess_id"); - if (raw_tess_id || tess_id) continue; + if (raw_tess_id || tess_id) continue; char *label = psMetadataLookupStr(&status, md, "label"); @@ -244,5 +244,5 @@ if (!status) { - psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); + psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id); psFree(output); return false; @@ -455,6 +455,6 @@ if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, "fakeRun"); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -1157,10 +1157,11 @@ // pxwarpQueueByFakeID() can only be run after fakeRun.state has been set to stop if (!pxwarpQueueByFakeID(config, - fakeRun->fake_id, - fakeRun->workdir, - fakeRun->label, - fakeRun->dvodb, - fakeRun->tess_id, - fakeRun->end_stage + fakeRun->fake_id, + fakeRun->workdir, + fakeRun->label, + fakeRun->dvodb, + fakeRun->tess_id, + fakeRun->reduction, + fakeRun->end_stage )) { psError(PS_ERR_UNKNOWN, false, "failed to queue warpRun"); Index: branches/eam_branches/20090522/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/magicdstool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/magicdstool.c (revision 24557) @@ -38,4 +38,6 @@ static bool revertdestreakedfileMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); +static bool toremoveMode(pxConfig *config); +static bool torestoreMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -68,4 +70,6 @@ MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); + MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); + MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); default: psAbort("invalid option (this should not happen)"); @@ -501,11 +505,11 @@ // chose the appropriate query based on the stage if (!strcmp(stage, "raw")) { - query = "UPDATE rawImfile SET magicked = 1 where exp_id = %" PRId64 " AND class_id = '%s'"; + query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'"; } else if (!strcmp(stage, "chip")) { - query = "UPDATE chipProcessedImfile SET magicked = 1 where chip_id = %" PRId64 " AND class_id = '%s'"; + query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'"; } else if (!strcmp(stage, "warp")) { - query = "UPDATE warpSkyfile SET magicked = 1 where warp_id = %" PRId64 " AND skycell_id = '%s'"; + query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'"; } else if (!strcmp(stage, "diff")) { - query = "UPDATE diffSkyfile SET magicked = 1 where diff_id = %" PRId64 " AND skycell_id = '%s'"; + query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'"; } else { psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); @@ -513,5 +517,5 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, query, stage_id, component)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -563,11 +567,11 @@ // chose the appropriate query based on the stage if (!strcmp(stage, "raw")) { - query = "UPDATE rawExp SET magicked = 1 where exp_id = %" PRId64; + query = "UPDATE rawExp SET magicked = %" PRId64 " where exp_id = %" PRId64; } else if (!strcmp(stage, "chip")) { - query = "UPDATE chipRun SET magicked = 1 where chip_id = %" PRId64; + query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64; } else if (!strcmp(stage, "warp")) { - query = "UPDATE warpRun SET magicked = 1 where warp_id = %" PRId64; + query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64; } else if (!strcmp(stage, "diff")) { - query = "UPDATE diffRun SET magicked = 1 where diff_id = %" PRId64; + query = "UPDATE diffRun SET magicked = %" PRId64 " where diff_id = %" PRId64; } else { psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage); @@ -575,5 +579,5 @@ return false; } - if (!p_psDBRunQueryF(config->dbh, query, stage_id)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; @@ -625,5 +629,5 @@ } - if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault)) { + if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault, "full")) { // rollback if (!psDBRollback(config->dbh)) { @@ -889,2 +893,148 @@ } +static bool toremoveMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magicdstool_toremove.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool torestoreMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magicdstool_torestore.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090522/ippTools/src/magicdstool.h =================================================================== --- branches/eam_branches/20090522/ippTools/src/magicdstool.h (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/magicdstool.h (revision 24557) @@ -32,4 +32,6 @@ MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, MAGICDSTOOL_MODE_GETSKYCELLS, + MAGICDSTOOL_MODE_TOREMOVE, + MAGICDSTOOL_MODE_TORESTORE, } MAGICDStoolMode; Index: branches/eam_branches/20090522/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/magicdstoolConfig.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/magicdstoolConfig.c (revision 24557) @@ -114,4 +114,20 @@ psMetadataAddBool(getskycellsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -toremove + psMetadata *toremoveArgs = psMetadataAlloc(); + psMetadataAddS64(toremoveArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(toremoveArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(toremoveArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -torestore + psMetadata *torestoreArgs = psMetadataAlloc(); + psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psFree(now); @@ -133,4 +149,8 @@ PXOPT_ADD_MODE("-getskycells", "get skycell files ", MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsArgs); + PXOPT_ADD_MODE("-toremove", "backup images pending removal", + MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs); + PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels", + MAGICDSTOOL_MODE_TORESTORE, torestoreArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090522/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/magictool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/magictool.c (revision 24557) @@ -47,9 +47,4 @@ static bool revertmaskMode(pxConfig *config); static bool maskMode(pxConfig *config); -static bool diffskyfileMode(pxConfig *config); -static bool warpskyfileMode(pxConfig *config); -static bool chipprocessedimfileMode(pxConfig *config); -static bool rawimfileMode(pxConfig *config); - static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -90,8 +85,4 @@ MODECASE(MAGICTOOL_MODE_REVERTMASK, revertmaskMode); MODECASE(MAGICTOOL_MODE_MASK, maskMode); - MODECASE(MAGICTOOL_MODE_DIFFSKYFILE, diffskyfileMode); - MODECASE(MAGICTOOL_MODE_WARPSKYFILE, warpskyfileMode); - MODECASE(MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileMode); - MODECASE(MAGICTOOL_MODE_RAWIMFILE, rawimfileMode); default: psAbort("invalid option (this should not happen)"); @@ -1306,266 +1297,4 @@ } - -static bool diffskyfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_diffskyfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool warpskyfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_warpskyfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicWarpSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool chipprocessedimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_chipprocessedimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicChipProcessedImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool rawimfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magictool_rawimfile.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicRawImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state) { Index: branches/eam_branches/20090522/ippTools/src/magictoolConfig.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/magictoolConfig.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/magictoolConfig.c (revision 24557) @@ -160,4 +160,5 @@ psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +#ifdef notdef // -diffskyfile psMetadata *diffskyfileArgs = psMetadataAlloc(); @@ -184,4 +185,5 @@ psMetadataAddStr(rawimfileArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); psMetadataAddBool(rawimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +#endif // notdef psFree(now); @@ -206,8 +208,10 @@ PXOPT_ADD_MODE("-revertmask", "", MAGICTOOL_MODE_REVERTMASK, revertmaskArgs); PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); +#ifdef notdef PXOPT_ADD_MODE("-diffskyfile", "", MAGICTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); PXOPT_ADD_MODE("-warpskyfile", "", MAGICTOOL_MODE_WARPSKYFILE, warpskyfileArgs); PXOPT_ADD_MODE("-chipprocessedimfile", "", MAGICTOOL_MODE_CHIPPROCESSEDIMFILE, chipprocessedimfileArgs); PXOPT_ADD_MODE("-rawimfile", "", MAGICTOOL_MODE_RAWIMFILE, rawimfileArgs); +#endif if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090522/ippTools/src/pubtool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/pubtool.c (revision 24557) +++ branches/eam_branches/20090522/ippTools/src/pubtool.c (revision 24557) @@ -0,0 +1,336 @@ +/* + * pubtool.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pubtool.h" + +static bool defineclientMode(pxConfig *config); +static bool definerunMode(pxConfig *config); +static bool pendingMode(pxConfig *config); +static bool addMode(pxConfig *config); +static bool revertMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = pubtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(PUBTOOL_MODE_DEFINECLIENT, defineclientMode); + MODECASE(PUBTOOL_MODE_DEFINERUN, definerunMode); + MODECASE(PUBTOOL_MODE_PENDING, pendingMode); + MODECASE(PUBTOOL_MODE_ADD, addMode); + MODECASE(PUBTOOL_MODE_REVERT, revertMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +static bool defineclientMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(product, config->args, "-product", true, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false); + + if (!publishClientInsert(config->dbh, 0, product, stage, workdir, comment)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + +static bool definerunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + + // required + + // optional + PXOPT_COPY_S64(config->args, where, "-client_id", "client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + psString query = pxDataGet("pubtool_definerun.sql"); // Query to run + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(where); + return false; + } + + psString whereClause = psStringCopy(""); // Additional constraints to add to query + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\n AND %s", clause); + psFree(clause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + psFree(whereClause); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + psFree(query); + psFree(whereClause); + + psArray *output = p_psDBFetchResult(config->dbh); // Output of SELECT statement + if (!output) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("pubtool", PS_LOG_INFO, "No rows found"); + psFree(output); + return true; + } + + for (int i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row of interest + psS64 client = psMetadataLookupS64(NULL, row, "client_id"); // Client identifier + psS64 stage = psMetadataLookupS64(NULL, row, "stage_id"); // Stage identifier + + if (!publishRunInsert(config->dbh, 0, client, stage, label, "new")) { + psError(PS_ERR_UNKNOWN, false, "Unable to add fileset"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + +static bool pendingMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + + // required + PXOPT_COPY_STR(config->args, where, "-client_id", "publishClient.client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "publishClient.stage", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "=="); + + // optional + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pubtool_pending.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psStringCopy(""); // WHERE conditions to add + if (psListLength(where->list)) { + psString clause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereClause, "\nAND %s", clause); + psFree(clause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(whereClause); + psFree(query); + return false; + } + psFree(whereClause); + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pubtool", PS_LOG_INFO, "No rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "publishRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "Failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool addMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add file"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + + if (fault == 0) { + if (!p_psDBRunQueryF(config->dbh, + "UPDATE publishRun SET state = 'full' WHERE pub_id = %" PRId64, + pub_id)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + } + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + return false; + } + + return true; +} + + + + +static bool revertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); // WHERE conditions + PXOPT_COPY_S64(config->args, where, "-pub_id", "publishRun.pub_id", "=="); + PXOPT_COPY_S32(config->args, where, "-fault", "publishDone.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-client_id", "publishClient.client_id", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "publishClient.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "publishRun.label", "=="); + + psString query = pxDataGet("pubtool_revert.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "Database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + Index: branches/eam_branches/20090522/ippTools/src/pubtool.h =================================================================== --- branches/eam_branches/20090522/ippTools/src/pubtool.h (revision 24557) +++ branches/eam_branches/20090522/ippTools/src/pubtool.h (revision 24557) @@ -0,0 +1,36 @@ +/* + * pubtool.h + * + * Copyright (C) 2008 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PUBTOOL_H +#define PUBTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + PUBTOOL_MODE_NONE = 0x0, + PUBTOOL_MODE_DEFINECLIENT, + PUBTOOL_MODE_DEFINERUN, + PUBTOOL_MODE_PENDING, + PUBTOOL_MODE_ADD, + PUBTOOL_MODE_REVERT, +} pubtoolMode; + +pxConfig *pubtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // PUBTOOL_H Index: branches/eam_branches/20090522/ippTools/src/pubtoolConfig.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/pubtoolConfig.c (revision 24557) +++ branches/eam_branches/20090522/ippTools/src/pubtoolConfig.c (revision 24557) @@ -0,0 +1,111 @@ +/* + * pubtoolConfig.c + * + * Copyright (C) 2008 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include "pxtools.h" +#include "pubtool.h" + +pxConfig *pubtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!"); + psFree(config); + return NULL; + } + + // -defineclient + psMetadata *defineclientArgs = psMetadataAlloc(); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(defineclientArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); + + // -definerun + psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-client_id", 0, "search by client_id", 0); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "set and search by label", NULL); + + // -pending + psMetadata *pendingArgs = psMetadataAlloc(); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-stage", 0, "search on source", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use simple output format?", false); + psMetadataAddU64(pendingArgs, PS_LIST_TAIL, "-limit", 0, "limit result set", 0); + + // -add + psMetadata *addArgs = psMetadataAlloc(); + psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0); + psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); + psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + + // -revert + psMetadata *revertArgs = psMetadataAlloc(); + psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-pub_id", 0, "search on pub_id", 0); + psMetadataAddS32(revertArgs, PS_LIST_TAIL, "-fault", 0, "search on fault code", 0); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-client_id", 0, "search on client_id", NULL); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL); + psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-defineclient", "", PUBTOOL_MODE_DEFINECLIENT, defineclientArgs); + PXOPT_ADD_MODE("-definerun", "", PUBTOOL_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-pending", "", PUBTOOL_MODE_PENDING, pendingArgs); + PXOPT_ADD_MODE("-add", "", PUBTOOL_MODE_ADD, addArgs); + PXOPT_ADD_MODE("-revert", "", PUBTOOL_MODE_REVERT, revertArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, true, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define database handle, if used + // do this last so we don't setup a connection before CLI options are validated + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/20090522/ippTools/src/pxchip.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/pxchip.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/pxchip.c (revision 24557) @@ -175,5 +175,5 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET state = '%s'"); - if (where) { + if (where && psListLength(where->list) > 0) { psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(&query, " %s", whereClause); @@ -216,5 +216,5 @@ psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); - if (where) { + if (where && psListLength(where->list) > 0) { psString whereClause = psDBGenerateWhereSQL(where, NULL); psStringAppend(&query, " %s", whereClause); @@ -264,5 +264,5 @@ dvodb, tess_id, - end_stage, + end_stage, 0 // magicked ) @@ -277,5 +277,5 @@ psS64 chip_id = psDBLastInsertID(config->dbh); - // Create rows in chipImfile table for each input exposure. + // Create rows in chipImfile table for each input exposure. // This creates the chip_image_id values psString query = "INSERT INTO chipImfile " @@ -295,5 +295,5 @@ return 0; } - + return chip_id; } Index: branches/eam_branches/20090522/ippTools/src/pxwarp.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/pxwarp.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/pxwarp.c (revision 24557) @@ -126,10 +126,11 @@ bool pxwarpQueueByFakeID(pxConfig *config, - psS64 fake_id, - char *workdir, - char *label, - char *dvodb, - char *tess_id, - char *end_stage) + psS64 fake_id, + const char *workdir, + const char *label, + const char *dvodb, + const char *tess_id, + const char *reduction, + const char *end_stage) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -146,4 +147,5 @@ dvodb, tess_id, + reduction, end_stage, NULL, // registered Index: branches/eam_branches/20090522/ippTools/src/pxwarp.h =================================================================== --- branches/eam_branches/20090522/ippTools/src/pxwarp.h (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/pxwarp.h (revision 24557) @@ -31,10 +31,11 @@ bool pxwarpQueueByFakeID(pxConfig *config, - psS64 fake_id, - char *workdir, - char *label, - char *dvodb, - char *tess_id, - char *end_stage); + psS64 fake_id, + const char *workdir, + const char *label, + const char *dvodb, + const char *tess_id, + const char *reduction, + const char *end_stage); #endif // PXWARP_H Index: branches/eam_branches/20090522/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/warptool.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/warptool.c (revision 24557) @@ -38,4 +38,5 @@ static bool tooverlapMode(pxConfig *config); static bool addoverlapMode(pxConfig *config); +static bool revertoverlapMode(pxConfig *config); static bool scmapMode(pxConfig *config); static bool towarpedMode(pxConfig *config); @@ -86,4 +87,5 @@ MODECASE(WARPTOOL_MODE_TOOVERLAP, tooverlapMode); MODECASE(WARPTOOL_MODE_ADDOVERLAP, addoverlapMode); + MODECASE(WARPTOOL_MODE_REVERTOVERLAP, revertoverlapMode); MODECASE(WARPTOOL_MODE_SCMAP, scmapMode); MODECASE(WARPTOOL_MODE_TOWARPED, towarpedMode); @@ -137,4 +139,5 @@ PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS) + PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // required (no default TESS) PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -157,7 +160,8 @@ dvodb, tess_id, + reduction, end_stage, registered, - false // magicked + 0 // magicked ); if (!warpRun) { @@ -247,4 +251,5 @@ PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); @@ -308,13 +313,12 @@ } - if (!tess_id && !row->tess_id) { - psError(PS_ERR_UNKNOWN, false, "cannot queue warp run without a defined tess id: label: %s, fake_id %" PRId64, row->label, row->fake_id); + if (!tess_id && !row->tess_id) { + psError(PS_ERR_UNKNOWN, false, "cannot queue warp run without a defined tess id: label: %s, fake_id %" PRId64, row->label, row->fake_id); psFree(output); return false; - } + } psFree(row); } - psFree(output); // loop over our list of fakeRun rows @@ -331,10 +335,11 @@ // queue the exp if (!pxwarpQueueByFakeID(config, - row->fake_id, - workdir ? workdir : row->workdir, - label ? label : row->label, - dvodb ? dvodb : row->dvodb, - tess_id ? tess_id : row->tess_id, - end_stage ? end_stage : row->end_stage)) + row->fake_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + dvodb ? dvodb : row->dvodb, + tess_id ? tess_id : row->tess_id, + reduction ? reduction : row->reduction, + end_stage ? end_stage : row->end_stage)) { psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); @@ -651,7 +656,7 @@ warpSkyCellMapInsert(config->dbh, warp_id, - NULL, // skycell_id - NULL, // tess_id - NULL, // class_id + "faulted", // skycell_id + "faulted", // tess_id + "faulted", // class_id fault // fault ); @@ -666,4 +671,60 @@ return true; } + +static bool revertoverlapMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyCellMap.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyCellMap.fault", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + int numDeleted; // Number deleted + { + psString query = pxDataGet("warptool_revertoverlap.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + numDeleted = psDBAffectedRows(config->dbh); + } + + psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkycellMap", numDeleted); + + psFree(where); + + return true; +} + @@ -919,5 +980,5 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false); + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); // default values @@ -1023,5 +1084,5 @@ return false; } - psS32 magicked = psMetadataLookupS64(&status, row, "magicked"); + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); if (!status) { psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked"); @@ -1224,53 +1285,4 @@ } - - // Update state to 'new' - int numUpdated; // Number updated - { - // This query is no longer necessary because we do not set warpRun's to full statte - // if they have faulted skyfiles. - // We do have runs in the DB that follow the old convention so we leave this in for - // now - psString query = pxDataGet("warptool_revertwarped_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numUpdated = psDBAffectedRows(config->dbh); - -#ifdef notdef - // new warpRuns won't get changed (they're already new) so don't require an update - if (numUpdated < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } -#endif - } - - psLogMsg("warptool", PS_LOG_INFO, "Updated %d warp runs", numUpdated); - // Delete product int numDeleted; // Number deleted @@ -1744,5 +1756,5 @@ if (clean) { - bool success = true; + bool success = true; if (!strcmp(tables[i].tableName, "warpRun")) { success = pxSetStateCleaned("warpRun", "state", output); Index: branches/eam_branches/20090522/ippTools/src/warptool.h =================================================================== --- branches/eam_branches/20090522/ippTools/src/warptool.h (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/warptool.h (revision 24557) @@ -34,4 +34,5 @@ WARPTOOL_MODE_TOOVERLAP, WARPTOOL_MODE_ADDOVERLAP, + WARPTOOL_MODE_REVERTOVERLAP, WARPTOOL_MODE_SCMAP, WARPTOOL_MODE_TOWARPED, Index: branches/eam_branches/20090522/ippTools/src/warptoolConfig.c =================================================================== --- branches/eam_branches/20090522/ippTools/src/warptoolConfig.c (revision 24529) +++ branches/eam_branches/20090522/ippTools/src/warptoolConfig.c (revision 24557) @@ -98,4 +98,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -111,5 +112,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tess_id", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tess_id (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-end_stage", 0, "define end stage", NULL); psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -156,4 +158,14 @@ psMetadataAddS64(addoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "set warp ID", 0); psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + // -revertoverlap + psMetadata *revertoverlapArgs = psMetadataAlloc(); + psMetadataAddS64(revertoverlapArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); + psMetadataAddStr(revertoverlapArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddStr(revertoverlapArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(revertoverlapArgs, PS_LIST_TAIL, "-label", 0, "search by warpRun label", NULL); + psMetadataAddS16(revertoverlapArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertoverlapArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + // -scmap @@ -300,4 +312,5 @@ PXOPT_ADD_MODE("-tooverlap", "", WARPTOOL_MODE_TOOVERLAP, tooverlapArgs); PXOPT_ADD_MODE("-addoverlap", "", WARPTOOL_MODE_ADDOVERLAP, addoverlapArgs); + PXOPT_ADD_MODE("-revertoverlap", "", WARPTOOL_MODE_REVERTOVERLAP, revertoverlapArgs); PXOPT_ADD_MODE("-scmap", "", WARPTOOL_MODE_SCMAP, scmapArgs); PXOPT_ADD_MODE("-towarped", "", WARPTOOL_MODE_TOWARPED, towarpedArgs); Index: branches/eam_branches/20090522/ippconfig/gpc1/ppImage.config =================================================================== --- branches/eam_branches/20090522/ippconfig/gpc1/ppImage.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/gpc1/ppImage.config (revision 24557) @@ -17,4 +17,86 @@ # use the deburned image instead of the raw, if it exists USE.DEBURNED.IMAGE BOOL TRUE # use burntool-repaired image? +TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool + +TILTYSTREAK.BY.CLASS METADATA # apply the 'tiltystreak' tool +APPLY.XY01 BOOL FALSE +APPLY.XY03 BOOL FALSE +APPLY.XY04 BOOL FALSE +APPLY.XY05 BOOL FALSE +APPLY.XY06 BOOL FALSE +APPLY.XY10 BOOL FALSE +APPLY.XY11 BOOL FALSE +APPLY.XY13 BOOL FALSE +APPLY.XY14 BOOL FALSE +APPLY.XY15 BOOL FALSE +APPLY.XY16 BOOL FALSE +APPLY.XY17 BOOL FALSE +APPLY.XY20 BOOL FALSE +APPLY.XY21 BOOL FALSE +APPLY.XY23 BOOL FALSE +APPLY.XY24 BOOL FALSE +APPLY.XY25 BOOL FALSE +APPLY.XY26 BOOL FALSE +APPLY.XY30 BOOL FALSE +APPLY.XY31 BOOL FALSE +APPLY.XY33 BOOL FALSE +APPLY.XY34 BOOL FALSE +APPLY.XY35 BOOL FALSE +APPLY.XY36 BOOL FALSE +APPLY.XY37 BOOL FALSE +APPLY.XY40 BOOL FALSE +APPLY.XY41 BOOL FALSE +APPLY.XY42 BOOL FALSE +APPLY.XY43 BOOL FALSE +APPLY.XY44 BOOL FALSE +APPLY.XY46 BOOL FALSE +APPLY.XY50 BOOL FALSE +APPLY.XY52 BOOL FALSE +APPLY.XY53 BOOL FALSE +APPLY.XY54 BOOL FALSE +APPLY.XY56 BOOL FALSE +APPLY.XY60 BOOL FALSE +APPLY.XY62 BOOL FALSE +APPLY.XY63 BOOL FALSE +APPLY.XY64 BOOL FALSE +APPLY.XY66 BOOL FALSE +APPLY.XY71 BOOL FALSE +APPLY.XY72 BOOL FALSE +APPLY.XY73 BOOL FALSE +APPLY.XY74 BOOL FALSE +APPLY.XY76 BOOL FALSE + +APPLY.XY02 BOOL TRUE +APPLY.XY12 BOOL TRUE +APPLY.XY22 BOOL TRUE +APPLY.XY27 BOOL TRUE +APPLY.XY32 BOOL TRUE +APPLY.XY45 BOOL TRUE +APPLY.XY47 BOOL TRUE +APPLY.XY51 BOOL TRUE +APPLY.XY55 BOOL TRUE +APPLY.XY57 BOOL TRUE +APPLY.XY61 BOOL TRUE +APPLY.XY65 BOOL TRUE +APPLY.XY67 BOOL TRUE +APPLY.XY75 BOOL TRUE + +OPTIONS.XY02 STR mask=0001011100010111000101110001011100010111000101110001011100010111 # cols: 3,5,6,7 +OPTIONS.XY12 STR mask=0000011100000111000001110000011100000111111101111111011111110111 # col 5,6,7 (+ 0,1,2,3 for row > 4) +OPTIONS.XY22 STR mask=0001001100010011000100110001001100010011000100110001001100010011 # cols: 3,6,7 +OPTIONS.XY27 STR none +OPTIONS.XY32 STR mask=0011011100110111001101110011011100110111001101110011011100110111 # cols: 2,3,5,6,7 +OPTIONS.XY45 STR mask=0111111101111111011111111111111111111111111111111111111111111111 # all except 00, 01, 02 +OPTIONS.XY47 STR mask=1111111001111110011111100111111001111110011111100111111001111110 # cols: 2,3,4,5,6,7 (add 00) +OPTIONS.XY51 STR mask=0011101100111011001110110011101100111011001110110011101100111011 # cols: 2,3,4,6,7 +OPTIONS.XY55 STR none +OPTIONS.XY57 STR none +OPTIONS.XY61 STR none +OPTIONS.XY65 STR mask=0111011101110111011101110111011101110111011101110111011101110111 # cols: 1,2,3,5,6,7 +OPTIONS.XY67 STR mask=1100111111001111110011111100111111001111110011111100111111001111 # cols: 0,1,4,5,6,7 +OPTIONS.XY75 STR mask=0001111100011111000111110001111100011111000111110001111100011111 # cols: 3,4,5,6,7 + +# OPTIONS.XY02 STR mask=00010111.00010111.00010111.00010111.00010111.00010111.00010111.00010111 # cols: 3,5,6,7 +END # crosstalk measurements : only valid for GPC1 for now @@ -44,4 +126,6 @@ END ASTROM METADATA + END + NOISEMAP METADATA END END Index: branches/eam_branches/20090522/ippconfig/gpc1/ppStack.config =================================================================== --- branches/eam_branches/20090522/ippconfig/gpc1/ppStack.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/gpc1/ppStack.config (revision 24557) @@ -32,2 +32,7 @@ TEMP.VARIANCE STR conv.var.fits # Suffix for convolved variance maps TEMP.DELETE BOOL FALSE # Delete temporary files on completion? + +PR METADATA + CONVOLVE BOOL FALSE + IMAGE.REJ F32 0.5 +END Index: branches/eam_branches/20090522/ippconfig/gpc1/psastro.config =================================================================== --- branches/eam_branches/20090522/ippconfig/gpc1/psastro.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/gpc1/psastro.config (revision 24557) @@ -189,2 +189,6 @@ # GHOST_OUTER_MINOR_0 F32 133.9 # GHOST_OUTER_MINOR_SLOPE F32 -0.00288 + +PR_RECIPE METADATA +REFSTAR_MASK BOOL FALSE +END Index: branches/eam_branches/20090522/ippconfig/gpc1/psphot.config =================================================================== --- branches/eam_branches/20090522/ippconfig/gpc1/psphot.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/gpc1/psphot.config (revision 24557) @@ -44,5 +44,5 @@ AP_MIN_SN F32 5.0 -OUTPUT.FORMAT STR PS1_V1 +OUTPUT.FORMAT STR PS1_V2 PSF_SHAPE_NSIGMA F32 3.0 # max significance for shape variation Index: branches/eam_branches/20090522/ippconfig/recipes/Makefile.am =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/Makefile.am (revision 24557) @@ -24,4 +24,5 @@ jpeg.mdc \ ppStatsFromMetadata.config \ + ppNoiseMap.config \ ppSkycell.config Index: branches/eam_branches/20090522/ippconfig/recipes/filerules-split.mdc =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/filerules-split.mdc (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/filerules-split.mdc (revision 24557) @@ -31,4 +31,5 @@ PPIMAGE.MASK INPUT @DETDB CHIP MASK PPIMAGE.BIAS INPUT @DETDB CHIP IMAGE +PPIMAGE.NOISEMAP INPUT @DETDB CHIP IMAGE PPIMAGE.DARK INPUT @DETDB CHIP DARK PPIMAGE.FLAT INPUT @DETDB CHIP IMAGE @@ -104,4 +105,7 @@ PPSIM.INPUT INPUT @FILES CHIP IMAGE PPSIM.REAL.SOURCES INPUT @FILES CHIP CMF + +## files used by ppImage +PPNOISEMAP.INPUT INPUT @FILES CHIP IMAGE ### output file definitions @@ -222,5 +226,5 @@ PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF -PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE +PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT PPSIM.FAKE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.fake.fits IMAGE NONE CHIP TRUE NONE PPSIM.FORCE.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.force.fits IMAGE NONE CHIP TRUE NONE @@ -241,4 +245,5 @@ TRACE.EXP OUTPUT {OUTPUT}.trace TEXT NONE FPA TRUE NONE +PPNOISEMAP.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE NONE # {FPA.OBS} Index: branches/eam_branches/20090522/ippconfig/recipes/ppImage.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/ppImage.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/ppImage.config (revision 24557) @@ -6,4 +6,5 @@ NONLIN BOOL FALSE # Non-linearity correction; not implemented OVERSCAN BOOL TRUE # Overscan subtraction +NOISEMAP BOOL FALSE # Apply read noise map BIAS BOOL TRUE # Bias subtraction DARK BOOL TRUE # Dark subtraction @@ -23,4 +24,8 @@ CHECK.CTE BOOL FALSE # measure CTE errors? USE.DEBURNED.IMAGE BOOL FALSE # use burntool-repaired image? +TILTYSTREAK.APPLY BOOL FALSE # apply the 'tiltystreak' tool + +TILTYSTREAK.BY.CLASS METADATA # apply the 'tiltystreak' tool +END # output data formats to save Index: branches/eam_branches/20090522/ippconfig/recipes/ppNoiseMap.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/ppNoiseMap.config (revision 24557) +++ branches/eam_branches/20090522/ippconfig/recipes/ppNoiseMap.config (revision 24557) @@ -0,0 +1,3 @@ + +XBIN S32 60 +YBIN S32 60 Index: branches/eam_branches/20090522/ippconfig/recipes/ppStack.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/ppStack.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/ppStack.config (revision 24557) @@ -3,5 +3,5 @@ CONVOLVE BOOL TRUE # Convolve images when stacking? ITER S32 1 # Number of rejection iterations -COMBINE.REJ F32 3.0 # Rejection threshold in combination (sigma) +COMBINE.REJ F32 2.0 # Rejection threshold in combination (sigma) COMBINE.SYS F32 0.03 # Relative systematic error in combination COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean @@ -13,5 +13,5 @@ DECONV.LIMIT F32 5.0 # Deconvolution fraction for rejecting entire image IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image -MATCH.REJ F32 2.5 # Rejection threshold for chi^2 values from matching +MATCH.REJ F32 5.0 # Rejection threshold for chi^2 values from matching ROWS S32 64 # Number of rows to read at once VARIANCE BOOL TRUE # Use variance in rejection? @@ -58,5 +58,5 @@ PSF.MODEL STR PS_MODEL_PS1_V1 # Model for PSF generation -TEMP.DIR STR /tmp # Directory for temporary images +TEMP.DIR STR /tmp # Directory for temporary images TEMP.IMAGE STR conv.im.fits # Suffix for temporary convolved images TEMP.MASK STR conv.mk.fits # Suffix for temporary convolved masks @@ -74,2 +74,8 @@ TEMP.DELETE BOOL FALSE # Delete temporary files on completion? END + +# Faster processing for PR images +PR METADATA + CONVOLVE BOOL FALSE + IMAGE.REJ F32 0.5 +END Index: branches/eam_branches/20090522/ippconfig/recipes/ppSub.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/ppSub.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/ppSub.config (revision 24557) @@ -6,5 +6,5 @@ REGION.SIZE F32 0 # Iso-kernel region size (pixels) SOURCE.RADIUS F32 3.0 # Source matching radius (pixels) -STAMP.SPACING F32 200 # Typical spacing between stamps (pixels) +STAMP.SPACING F32 400 # Typical spacing between stamps (pixels) STAMP.FOOTPRINT S32 20 # Size of stamps (pixels) STAMP.THRESHOLD F32 5 # Flux threshold for stamps (stdev above background) @@ -44,7 +44,8 @@ INTERPOLATION STR LANCZOS3 # Interpolation mode for bad pixels +DUAL BOOL FALSE # Dual convolution? INVERSE BOOL FALSE # Generate inverse subtraction? PHOTOMETRY BOOL FALSE # Perform photometry? - +SAVE.CONVOLVED BOOL TRUE # Save convolved images? # Recipe overrides for STACK @@ -66,2 +67,7 @@ PHOTOMETRY BOOL TRUE # Perform photometry? END + +#PR image class +PR METADATA + PHOTOMETRY BOOL FALSE +END Index: branches/eam_branches/20090522/ippconfig/recipes/psastro.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/psastro.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/psastro.config (revision 24557) @@ -178,2 +178,8 @@ PSASTRO.EXTRACT METADATA END + +DEFAULT_RECIPE METADATA +END + +PR_RECIPE METADATA +END Index: branches/eam_branches/20090522/ippconfig/recipes/psphot.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/psphot.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/psphot.config (revision 24557) @@ -18,4 +18,6 @@ ZERO_POINT F32 25.000 # zero point used by DVO ZERO_PT F32 25.000 # zero point used by DVO + +OUTPUT.FORMAT STR PS1_V2 # these parameter govern how the background is measured @@ -286,2 +288,11 @@ DIFF METADATA END + + +# Recipe overrides for PR reduction +PR METADATA + FULL_FIT_SN_LIM F32 100.0 + BACKGROUND.XBIN S32 2400 + BACKGROUND.YBIN S32 2400 + IMSTATS_NPIX S32 50000 +END Index: branches/eam_branches/20090522/ippconfig/recipes/pswarp.config =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/pswarp.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/pswarp.config (revision 24557) @@ -10,2 +10,6 @@ ACCEPT.FRAC F32 0.1 # Minimum fraction of good pixels to accept result INTERPOLATION.NUM S32 1000 # Number of interpolation kernels to pre-calculate + +# Default recipe for warping +WARP METADATA +END Index: branches/eam_branches/20090522/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/eam_branches/20090522/ippconfig/recipes/reductionClasses.mdc (revision 24529) +++ branches/eam_branches/20090522/ippconfig/recipes/reductionClasses.mdc (revision 24557) @@ -130,13 +130,15 @@ CHIP_PPIMAGE STR CHIP CHIP_PSPHOT STR CHIP - STACK_PPSTACK STR STACK - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK - DIFF_PPSUB STR DIFF - DIFF_PSPHOT STR DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - FAKEPHOT STR FAKEPHOT - ADDSTAR STR ADDSTAR + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE END @@ -145,13 +147,15 @@ CHIP_PPIMAGE STR PPIMAGE_P CHIP_PSPHOT STR CHIP - STACK_PPSTACK STR STACK - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK - DIFF_PPSUB STR DIFF - DIFF_PSPHOT STR DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - FAKEPHOT STR FAKEPHOT - ADDSTAR STR ADDSTAR + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE END @@ -160,4 +164,5 @@ CHIP_PPIMAGE STR PPIMAGE_FLATCORR CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP STACK_PPSTACK STR STACK STACK_PPSUB STR STACK @@ -169,4 +174,5 @@ FAKEPHOT STR FAKEPHOT ADDSTAR STR FLATCORR + PSASTRO STR DEFAULT_RECIPE END @@ -175,4 +181,5 @@ CHIP_PPIMAGE STR PPIMAGE_FLATTEST CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP STACK_PPSTACK STR STACK STACK_PPSUB STR STACK @@ -184,4 +191,5 @@ FAKEPHOT STR FAKEPHOT ADDSTAR STR FLATCORR + PSASTRO STR DEFAULT_RECIPE END @@ -190,12 +198,14 @@ CHIP_PPIMAGE STR PPIMAGE_DET_ONLY CHIP_PSPHOT STR CHIP - STACK_PPSTACK STR STACK - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK - DIFF_PPSUB STR DIFF - DIFF_PSPHOT STR DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - FAKEPHOT STR FAKEPHOT + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + PSASTRO STR DEFAULT_RECIPE END @@ -203,12 +213,14 @@ CHIP_PPIMAGE STR PPIMAGE_OBDSF CHIP_PSPHOT STR CHIP - STACK_PPSTACK STR STACK - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK - DIFF_PPSUB STR DIFF - DIFF_PSPHOT STR DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - FAKEPHOT STR FAKEPHOT + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + PSASTRO STR DEFAULT_RECIPE END @@ -217,4 +229,5 @@ CHIP_PPIMAGE STR PPIMAGE_MASKPHOT CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP STACK_PPSTACK STR STACK STACK_PPSUB STR STACK @@ -232,21 +245,32 @@ CHIP_PPIMAGE STR CHIP_DENSE_STARS CHIP_PSPHOT STR CHIP - STACK_PPSTACK STR STACK - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK - DIFF_PPSUB STR DIFF - DIFF_PSPHOT STR DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - FAKEPHOT STR FAKEPHOT - ADDSTAR STR ADDSTAR -END - -# Intended for areas with high stellar density -WARPWARP METADATA - DIFF_PPSUB STR WARPWARP - DIFF_PSPHOT STR DIFF - JPEG_BIN1 STR PPIMAGE_J1 - JPEG_BIN2 STR PPIMAGE_J2 - ADDSTAR STR ADDSTAR -END + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE +END + + +# Intended for fast processing to make PR images +PR METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR PR + WARP_PSWARP STR WARP + STACK_PPSTACK STR PR + STACK_PPSUB STR PR + STACK_PSPHOT STR PR + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR PR_RECIPE +END Index: branches/eam_branches/20090522/ippconfig/system.config =================================================================== --- branches/eam_branches/20090522/ippconfig/system.config (revision 24529) +++ branches/eam_branches/20090522/ippconfig/system.config (revision 24557) @@ -51,4 +51,5 @@ PPSTATS_METADATA STR recipes/ppStatsFromMetadata.config # Image statistics DVOCORR STR recipes/dvocorr.config # Image statistics - PPSKYCELL STR recipes/ppSkycell.config # JPEGs of skycells + PPSKYCELL STR recipes/ppSkycell.config # JPEGs of skycells + PPNOISEMAP STR recipes/ppNoiseMap.config # used to create the noise map END Index: branches/eam_branches/20090522/magic/magic/makefile =================================================================== --- branches/eam_branches/20090522/magic/magic/makefile (revision 24529) +++ branches/eam_branches/20090522/magic/magic/makefile (revision 24557) @@ -1,32 +1,38 @@ -# -# This makefile is used in place of the one in Paul Sydneys' tarball -# -# XXX integrate with the build system -# -TARGETS = DetectStreaks -SSA_SRC_DIR = ../ssa-core-cpp/src -CFITSIO_DIR = ../cfitsio -CFITSIO_DIR = $(PSCONFDIR)/$(PSCONFIG) -INCLUDEDIRS = -I$(SSA_SRC_DIR) -I$(CFITSIO_DIR)/include -# INCLUDEDIRS = -I$(SSA_SRC_DIR) +TARGETS = DetectStreaks CXX = c++ -CXXFLAGS = -Wall -g -CXXFLAGS = -Wall -O3 -DNDEBUG +CXXFLAGS = -W -Wall -O3 -DNDEBUG +# CXXFLAGS = -W -Wall -g -SRCS = $(TARGETS).cpp $(SSA_SRC_DIR)/math/Constants.cpp -LIBS = -L$(CFITSIO_DIR) -lcfitsio -LIBS = -L$(CFITSIO_DIR)/lib -lcfitsio -# LIBS = -lcfitsio +SSA_SRC_DIR = ../ssa-core-cpp/src +INCLUDEDIRS = -I. -I$(SSA_SRC_DIR) +OTHER_SRCS = $(SSA_SRC_DIR)/math/Constants.cpp -$(TARGETS) : $(SRCS) - $(CXX) $(CXXFLAGS) $(INCLUDEDIRS) $(SRCS) $(LIBS) -o $@ +# CFITSIO_DIR = ../cfitsio +CFITSIO_DIR = $(PSCONFDIR)/$(PSCONFIG) + +CFITSIO_INC = -I$(CFITSIO_DIR) +CFITSIO_LIB = -L$(CFITSIO_DIR) -lcfitsio + +all : $(TARGETS) + +RemoveStreaks : $(OTHER_SRCS) + +DetectStats : $(OTHER_SRCS) + +% :: %.cpp + $(CXX) $(CXXFLAGS) $(INCLUDEDIRS) $(CFITSIO_INC) $< $(OTHER_SRCS) $(CFITSIO_LIB) -o $@ + +.PHONY : clean + +.SILENT : clean clean : - -rm -f $(TARGETS) + - rm $(TARGETS) DESTDIR=$(PSCONFDIR)/$(PSCONFIG) -install: $(TARGETS) +install: $(TARGETS) cp DetectStreaks $(DESTDIR)/bin chmod 755 $(DESTDIR)/bin/DetectStreaks + Index: branches/eam_branches/20090522/magic/remove/src/Line.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/Line.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/Line.c (revision 24557) @@ -16,4 +16,13 @@ { double temp = *first; + *first = *second; + *second = temp; +} + +/** Internal routine to swap integer values */ + +void SwapInt (int* first, int* second) +{ + int temp = *first; *first = *second; *second = temp; @@ -255,15 +264,110 @@ } -/** Map a line to an image for its specified width and store as a list - of pixel positions +/** Clip the line between (minX,minY) and (maxX,maxY) + + @param[in,out] line line to be clipped within the bounds + @param[in] minX minimum X (columns) for the line + @param[in] minY minimum Y (rows) for the line + @param[in] maxX maximum X (columns) for the line + @param[in] maxY maximum Y (rows) for the line + @return true if line overlaps the clip boundaries */ + +bool LineClipFull (Line *line, int minX, int minY, int maxX, int maxY) +{ + unsigned int i, found = 0; + Line boundLine, clipLine; + strkPt tuple1, tuple2, vertices[4]; + vertices[0].x = minX; vertices[0].y = minY; + vertices[1].x = maxX; vertices[1].y = minY; + vertices[2].x = maxX; vertices[2].y = maxY; + vertices[3].x = minX; vertices[3].y = maxY; + + for (i = 0; i < 4 && found < 2; ++i) + { + boundLine.begin = vertices[i]; + boundLine.end = vertices[(i + 1) % 4]; + if (LineIntercept (line, &boundLine, &tuple1, &tuple2, false, true)) + { + if (found == 0) + { + clipLine.begin = tuple1; + ++found; + } + else if (tuple1.x != clipLine.begin.x || + tuple1.y != clipLine.begin.y) + { + clipLine.end = tuple1; + ++found; + } + } + } + + // If two endpoints are found, clip the line + + if (found > 1) + { + if (clipLine.begin.x <= clipLine.end.x) + { + line->begin = clipLine.begin; + line->end = clipLine.end; + } + else + { + line->begin = clipLine.end; + line->end = clipLine.begin; + } + } + return found > 1; +} + +/** Move a line by the specified X and Y offsets + @param[in,out] line Line to move by X and Y offsets + @param[in] xOffset X shift applied to both endpoints + @param[in] yOffset Y shift applied to both endpoints */ + +void LineMove (Line *line, double xOffset, double yOffset) +{ + line->begin.x += xOffset; + line->begin.y += yOffset; + line->end.x += xOffset; + line->end.y += yOffset; +} + +/** Return the maximum bounds between the line endpoints and + current bounds + @param[in] line Line endpoints to compare + @param[in,out] xMin minimum X value to update + @param[in,out] xMax maximum X value to update + @param[in,out] yMin minimum Y value to update + @param[in,out] yMax maximum Y value to update */ + +void MaxBounds (Line *line, int *xMin, int *xMax, int *yMin, int *yMax) +{ + if (line->begin.x < *xMin) *xMin = (int) floor (line->begin.x); + if (line->end.x < *xMin) *xMin = (int) floor (line->end.x); + if (line->begin.y < *yMin) *yMin = (int) floor (line->begin.y); + if (line->end.y < *yMin) *yMin = (int) floor (line->end.y); + + if (line->begin.x > *xMax) *xMax = (int) ceil (line->begin.x); + if (line->end.x > *xMax) *xMax = (int) ceil (line->end.x); + if (line->begin.y > *yMax) *yMax = (int) ceil (line->begin.y); + if (line->end.y > *yMax) *yMax = (int) ceil (line->end.y); +} + +/** Map a line to an image for its specified width and store as + a list of pixel positions @param[out] pixels list of PixelPos pointers corresponding based on the line settings - @param[in] line Line to map to pixels */ - -void PixelsFromLine (StreakPixels* pixels, Line *line) -{ + @param[in] line Line to map to pixels + @param[in] numCols maximum X (columns) for the line + @param[in] numRows maximum Y (rows) for the line */ + +void PixelsFromLine (StreakPixels* pixels, Line *line, int numCols, int numRows) +{ + Line offsetLine; PixelPos *pixel; - double slope, xOffset, yOffset, xMid, yMid, xBegin, yBegin, xEnd, yEnd, x, y; + double slope, xOffset, yOffset, xMid, yMid; + int x, y, xBegin = numCols, yBegin = numRows, xEnd = 0, yEnd = 0; // Extract the endpoints @@ -280,7 +384,22 @@ double dr = sqrt (dx * dx + dy * dy); double halfWidth = line->width / 2.0; - double halfWidth2 = halfWidth * halfWidth; if (!dr) return; - + + // Compute the intercepts of line width bounds and determine maximum + // bounds in each axis + + xOffset = -halfWidth * dy / dr; + yOffset = halfWidth * dx / dr; + + offsetLine = *line; + LineMove (&offsetLine, xOffset, yOffset); + if (LineClip (&offsetLine, numCols, numRows)) + MaxBounds (&offsetLine, &xBegin, &xEnd, &yBegin, &yEnd); + + offsetLine = *line; + LineMove (&offsetLine, -xOffset, -yOffset); + if (LineClip (&offsetLine, numCols, numRows)) + MaxBounds (&offsetLine, &xBegin, &xEnd, &yBegin, &yEnd); + // Step point by point based on the dominate axis @@ -307,25 +426,27 @@ // Compute the x and y offsets for the line width extent - xOffset = halfWidth * dy / dr; - yOffset = halfWidth * dr / dx; - yMid = y1 + slope * (floor (x1 - xOffset) - x1); - xBegin = floor (x1 - xOffset); - xEnd = ceil (x2 + xOffset) + 1.0; - - for (x = xBegin; x < xEnd; ++x) - { - yBegin = floor (yMid - yOffset); - yEnd = ceil (yMid + yOffset) + 1.0; - for (y = yBegin; y < yEnd; ++y) + if (xBegin > xEnd) + SwapInt (&xBegin, &xEnd); + else + ++xEnd; + if (xBegin < 0) xBegin = 0; + if (xEnd > numCols) xEnd = numCols; + + yMid = y1 + slope * (xBegin - x1); + yOffset = fabs (halfWidth * dr / dx); + + for (x = xBegin; x != xEnd; ++x) + { + yBegin = (int) floor (yMid - yOffset); + yEnd = (int) ceil (yMid + yOffset) + 1; + for (y = yBegin; y != yEnd; ++y) { - if (DistanceSquared (line, x, y) <= halfWidth2) + if (y >= 0 && y < numRows) { - if (x >=0 && y >= 0) { - pixel = psAlloc (sizeof(PixelPos)); - pixel->x = (unsigned int) x; - pixel->y = (unsigned int) y; - psArrayAdd (pixels, 1024, pixel); - psFree (pixel); - } + pixel = psAlloc (sizeof(PixelPos)); + pixel->x = (unsigned int) x; + pixel->y = (unsigned int) y; + psArrayAdd (pixels, 1024, pixel); + psFree (pixel); } } @@ -354,27 +475,28 @@ // Compute the x and y offsets for the line width extent - - xOffset = halfWidth * dr / dy; - yOffset = halfWidth * dx / dr; - - xMid = x1 + slope * (floor (y1 - yOffset) - y1); - yBegin = floor (y1 - yOffset); - yEnd = ceil (y2 + yOffset) + 1.0; - - for (y = yBegin; y < yEnd; ++y) - { - xBegin = floor (xMid - xOffset); - xEnd = ceil (xMid + xOffset) + 1.0; - for (x = xBegin; x < xEnd; ++x) + + if (yBegin > yEnd) + SwapInt (&yBegin, &yEnd); + else + ++yEnd; + if (yBegin < 0) yBegin = 0; + if (yEnd > numRows) yEnd = numRows; + + xMid = x1 + slope * (yBegin - y1); + xOffset = fabs (halfWidth * dr / dy); + + for (y = yBegin; y != yEnd; ++y) + { + xBegin = (int) floor (xMid - xOffset); + xEnd = (int) ceil (xMid + xOffset) + 1; + for (x = xBegin; x != xEnd; ++x) { - if (DistanceSquared (line, x, y) <= halfWidth2) + if (x >=0 && x < numCols) { - if (x >=0 && y >= 0) { - pixel = psAlloc (sizeof(PixelPos)); - pixel->x = (unsigned int) x; - pixel->y = (unsigned int) y; - psArrayAdd (pixels, 1024, pixel); - psFree (pixel); - } + pixel = psAlloc (sizeof(PixelPos)); + pixel->x = (unsigned int) x; + pixel->y = (unsigned int) y; + psArrayAdd (pixels, 1024, pixel); + psFree (pixel); } } Index: branches/eam_branches/20090522/magic/remove/src/Line.h =================================================================== --- branches/eam_branches/20090522/magic/remove/src/Line.h (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/Line.h (revision 24557) @@ -21,4 +21,15 @@ extern bool LineClip (Line *line, int numCols, int numRows); +/** Clip the line between (minX,minY) and (maxX,maxY) + + @param[in,out] line line to be clipped within the bounds + @param[in] minX minimum X (columns) for the line + @param[in] minY minimum Y (rows) for the line + @param[in] maxX maximum X (columns) for the line + @param[in] maxY maximum Y (rows) for the line + @return true if line overlaps the clip boundaries */ + +extern bool LineClipFull (Line *line, int minX, int minY, int maxX, int maxY); + /** Map a line to an image for its specified width and append as a list of pixel positions @@ -26,7 +37,10 @@ @param[out] pixels list of PixelPos pointers corresponding based on the line settings - @param[in] line Line to map to pixels */ + @param[in] line Line to map to pixels + @param[in] numCols maximum X (columns) for the line + @param[in] numRows maximum Y (rows) for the line */ -extern void PixelsFromLine (StreakPixels* pixels, Line *line); +extern void PixelsFromLine (StreakPixels* pixels, Line *line, + int numCols, int numRows); #endif /* STREAK_LINE_H */ Index: branches/eam_branches/20090522/magic/remove/src/Makefile.simple =================================================================== --- branches/eam_branches/20090522/magic/remove/src/Makefile.simple (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/Makefile.simple (revision 24557) @@ -28,5 +28,12 @@ streaksrelease.o -# STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1 +ISDESTREAKED_OBJECTS= \ + ${COMMON_OBJECTS} \ + isdestreaked.o + + +HEADERS= Line.h streaksastrom.h streaksextern.h streaksio.h streaksremove.h + +STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1 OPTFLAGS= -g -O2 OPTFLAGS= -g @@ -35,8 +42,10 @@ LDFLAGS=`psmodules-config --libs` -PROGRAMS= streaksremove streaksreplace streakscompare streaksrelease +PROGRAMS= streaksremove streaksreplace streakscompare streaksrelease isdestreaked +HEADERS=Line.h streaksastrom.h streaksextern.h streaksio.h streaksremove.h all: ${PROGRAMS} +${REMOVE_OBJECTS}: ${HEADERS} streaksremove: ${REMOVE_OBJECTS} @@ -47,4 +56,7 @@ streaksrelease: ${RELEASE_OBJECTS} +isdestreaked: ${ISDESTREAKED_OBJECTS} + + install: ${PROGRAMS} install -t $(PSCONFDIR)/$(PSCONFIG)/bin streaksremove @@ -52,4 +64,5 @@ install -t $(PSCONFDIR)/$(PSCONFIG)/bin streakscompare install -t $(PSCONFDIR)/$(PSCONFIG)/bin streaksrelease + install -t $(PSCONFDIR)/$(PSCONFIG)/bin isdestreaked clean: Index: branches/eam_branches/20090522/magic/remove/src/isdestreaked.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/isdestreaked.c (revision 24557) +++ branches/eam_branches/20090522/magic/remove/src/isdestreaked.c (revision 24557) @@ -0,0 +1,40 @@ +#include "streaksremove.h" + +int +main(int argc, char *argv[]) +{ + pmConfig *config = pmConfigRead(&argc, argv, NULL); + if (!config) { + psErrorStackPrint(stderr, "failed to parse arguments\n"); + exit(PS_EXIT_CONFIG_ERROR); + } + + char *fileName = argv[1]; + if (!fileName) { + fprintf(stderr, "Usage: %s filename\n", argv[0]); + exit(PS_EXIT_DATA_ERROR); + } + + psFits *fits = psFitsOpen(fileName, "r"); + if (!fits) { + psError(PS_ERR_UNKNOWN, false, "failed to open fits file: %s\n", fileName); + psErrorStackPrint(stderr, ""); + exit(PS_EXIT_DATA_ERROR); + return false; + } + + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psError(PS_ERR_IO, false, "failed to read header for: %s", fileName); + psErrorStackPrint(stderr, ""); + return false; + } + + bool mdok; + bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK"); + if (mdok && isDestreaked) { + exit(0); + } else { + exit(42); + } +} Index: branches/eam_branches/20090522/magic/remove/src/streaksastrom.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksastrom.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksastrom.c (revision 24557) @@ -150,4 +150,100 @@ bool +SkyToLocal(strkPt *outPt, strkAstrom *astrom, double ra, double dec) +{ + // generate a local project using the RA, DEC of the 0,0 pixel of the chip as the + // projection center with the same plate scale as the nominal TP->Sky astrometry. + + pmFPA *fpa = (pmFPA *) astrom->fpa; + pmChip *chip = (pmChip *) astrom->chip; + + // find the RA,DEC coords of the 0,0 pixel for this chip: + + psPlane ptTP; + psSphere ptSky; + + ptSky.r = ra; + ptSky.d = dec; + ptSky.rErr = 0.0; + ptSky.dErr = 0.0; + + psProject(&ptTP, &ptSky, fpa->toSky); + + outPt->x = ptTP.x; + outPt->y = ptTP.y; + + return true; +} + +bool +LocalToSky(strkPt *outPt, strkAstrom *astrom, strkPt *inPt) +{ + // generate a local project using the RA, DEC of the 0,0 pixel of the chip as the + // projection center with the same plate scale as the nominal TP->Sky astrometry. + + pmFPA *fpa = (pmFPA *) astrom->fpa; + pmChip *chip = (pmChip *) astrom->chip; + + // find the RA,DEC coords of the 0,0 pixel for this chip: + + psPlane ptTP; + psSphere ptSky; + + ptTP.x = inPt->x; + ptTP.y = inPt->y; + ptTP.xErr = 0.0; + ptTP.yErr = 0.0; + + psDeproject(&ptSky, &ptTP, fpa->toSky); + + outPt->x = ptSky.r; + outPt->y = ptSky.d; + + return true; +} + +bool +componentBounds(int *minX, int *minY, int *maxX, int *maxY, strkAstrom *astrom, int numCols, int numRows) +{ + // find the bounds of the (padded) chip region in tangent-plane coordinates + + pmFPA *fpa = (pmFPA *) astrom->fpa; + pmChip *chip = (pmChip *) astrom->chip; + + psPlane ptCH, ptFP, TPo, TPx, TPy; + + // coordinate of the chip center: + ptCH.x = 0.5*numCols; + ptCH.y = 0.5*numRows; + psPlaneTransformApply(&ptFP, chip->toFPA, &ptCH); + psPlaneTransformApply(&TPo, fpa->toTPA, &ptFP); + + // coordinate of the chip center + dX/2: + ptCH.x = numCols; + ptCH.y = 0.5*numRows; + psPlaneTransformApply(&ptFP, chip->toFPA, &ptCH); + psPlaneTransformApply(&TPx, fpa->toTPA, &ptFP); + + // coordinate of the chip center + dY/2: + ptCH.x = 0.5*numCols; + ptCH.y = numRows; + psPlaneTransformApply(&ptFP, chip->toFPA, &ptCH); + psPlaneTransformApply(&TPy, fpa->toTPA, &ptFP); + + // half-lengths of the two sides in tangent-plane coords: + double xSize = hypot (TPx.x - TPo.x, TPx.y - TPo.y); + double ySize = hypot (TPy.x - TPo.x, TPy.y - TPo.y); + double radius = hypot (xSize, ySize); + + // define the region encompassed by the radius with some padding: + *minX = TPo.x - 1.1*radius; + *minY = TPo.y - 1.1*radius; + *maxX = TPo.x + 1.1*radius; + *maxY = TPo.y + 1.1*radius; + + return true; +} + +bool skyToCell(strkPt *outPt, strkAstrom *astrom, double ra, double dec) { Index: branches/eam_branches/20090522/magic/remove/src/streaksastrom.h =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksastrom.h (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksastrom.h (revision 24557) @@ -32,3 +32,7 @@ extern void linearizeTransforms(strkAstrom *astrom); +extern bool SkyToLocal(strkPt *outPt, strkAstrom *astrom, double ra, double dec); +extern bool LocalToSky(strkPt *outPt, strkAstrom *astrom, strkPt *inPt); +extern bool componentBounds(int *minX, int *minY, int *maxX, int *maxY, strkAstrom *astrom, int numCols, int numRows); + #endif // STREAKS_ASTROM_H Index: branches/eam_branches/20090522/magic/remove/src/streaksextern.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksextern.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksextern.c (revision 24557) @@ -36,4 +36,10 @@ StreakPixels *pixels = psArrayAllocEmpty (1024); int streaksOnComponent = 0; + + int minX, minY, maxX, maxY; + + // find the chip dimensions in the tangent-plane coordinates (length of hypotenuse) + componentBounds (&minX, &minY, &maxX, &maxY, astrom, numCols, numRows); + for (i = 0; i != streaks->size; ++i) { @@ -41,11 +47,31 @@ line.width = streaks->list[i].width; - if (skyToCell (&line.begin, astrom, - streaks->list[i].ra1, streaks->list[i].dec1) && - skyToCell (&line.end, astrom, - streaks->list[i].ra2, streaks->list[i].dec2) && + + /* Use tangent plane coordinates to narrow down the ra,dec range of the line closer to + * the chip boundaries. Use these new ra,dec positions to generate the line on the + * chip using the full non-linear astrometry */ + + // project the ends of the line using a linear projection centered on the chip center: + Line full; + SkyToLocal (&full.begin, astrom, streaks->list[i].ra1, streaks->list[i].dec1); + SkyToLocal (&full.end, astrom, streaks->list[i].ra2, streaks->list[i].dec2); + + // clip the line to a square box with diameter = hypotenuse of the chip image centerd + // on the chip center in tangent-plane coordinates. skip the rest of this streak if + // the line does not intersect this region + if (!LineClipFull (&full, minX, minY, maxX, maxY)) { + continue; + } + + // convert the end points back into ra, dec pairs: + strkPt sky1, sky2; + LocalToSky (&sky1, astrom, &full.begin); + LocalToSky (&sky2, astrom, &full.end); + + if (skyToCell (&line.begin, astrom, sky1.x, sky1.y) && + skyToCell (&line.end, astrom, sky2.x, sky2.y) && LineClip (&line, numCols, numRows)) { - PixelsFromLine (pixels, &line); + PixelsFromLine (pixels, &line, numCols, numRows); streaksOnComponent++; } Index: branches/eam_branches/20090522/magic/remove/src/streaksio.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksio.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksio.c (revision 24557) @@ -19,4 +19,9 @@ psMemSetDeallocator(sf, (psFreeFunc) streakFilesFree); memset(sf, 0, sizeof(*sf)); + + if (remove) { + // remember pointer so that streaksExit can delete temps + setStreakFiles(sf); + } sf->config = config; @@ -160,4 +165,33 @@ } +// figure out if a nebulous instance is a non-destreaked file +static bool +nebFileIsDestreaked(sFile *sfile) +{ + if (!sfile->resolved_name) { + psError(PS_ERR_PROGRAMMING, true, "resolved name is null"); + return false; + } + psFits *fits = psFitsOpen(sfile->resolved_name, "r"); + if (!fits) { + psError(PS_ERR_IO, true, "failed open %s", sfile->name); + // can't tell if it is a destreaked file + return false; + } + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psError(PS_ERR_IO, true, "failed to read header for: %s", sfile->name); + return false; + } + bool mdok; + bool isDestreaked = psMetadataLookupBool(&mdok, header, "PSDESTRK"); + if (mdok && isDestreaked) { + return true; + } else { + psError(PS_ERR_IO, false, "output file already exists and may not be de-streaked: %s", sfile->name); + return false; + } +} + static psString resolveFilename(pmConfig *config, sFile *sfile, bool create) @@ -171,5 +205,11 @@ // delete the existing file, since there may be more than one // instance. It will get created below in pmConfigConvertFilename - if (nebFind(server, sfile->name)) { + if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) { + if (!nebFileIsDestreaked(sfile)) { + psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name); + return NULL; + } + nebFree(sfile->resolved_name); + sfile->resolved_name = NULL; nebDelete(server, sfile->name); } @@ -193,5 +233,5 @@ // all of the keywords in the raw image files written to the output destreaked files - if (!CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) { + if (!outputFilename && !CHIP_LEVEL_INPUT(stage) && !strcmp(fileSelect, "INPUT")) { // stage is warp or diff AND fileSelect eq "INPUT" // get data from pmFPAfile. @@ -250,5 +290,5 @@ } - // if outputFilename is not null name it contains the "directory" + // if outputFilename is not null name it contains the "directory" (perhaps with a prefix like SR_) // and outputFilename is the basename name of the file (or nebulous key) // and the file is to be opened for writing @@ -334,4 +374,18 @@ void +addDestreakKeyword(psMetadata *header) +{ + psMetadataAddBool(header, PS_LIST_TAIL, "PSDESTRK", PS_META_REPLACE, + "Have streaks been removed from image?", true); +} + +void +addRecoveryKeyword(psMetadata *header) +{ + psMetadataAddBool(header, PS_LIST_TAIL, "PSRECOVR", PS_META_REPLACE, + "Does this image contain excised streak pixels?", true); +} + +void copyPHU(streakFiles *sfiles, bool remove) { @@ -344,6 +398,13 @@ streaksExit("", PS_EXIT_DATA_ERROR); } - - // TODO: add keyword indicating that streaks have been removed + psMetadata *recHeader = NULL; + if (remove && sfiles->recImage) { + recHeader = psMetadataCopy(NULL, imageHeader); + addRecoveryKeyword(recHeader); + } + + // add keyword indicating that streaks have been removed + addDestreakKeyword(imageHeader); + if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", @@ -351,10 +412,11 @@ streaksExit("", PS_EXIT_DATA_ERROR); } - // TODO: add keyword indicating that this is the recovery image - if (remove && sfiles->recImage && !psFitsWriteBlank(sfiles->recImage->fits, imageHeader, NULL)) { + if (recHeader && !psFitsWriteBlank(sfiles->recImage->fits, recHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", sfiles->recImage->resolved_name); streaksExit("", PS_EXIT_DATA_ERROR); } + psFree(recHeader); + recHeader = NULL; psFree(imageHeader); @@ -367,5 +429,11 @@ streaksExit("", 1); } - // TODO: add keyword indicating that streaks have been removed + if (remove && sfiles->recMask) { + recHeader = psMetadataCopy(NULL, maskHeader); + // add keyword indicating that this is the recovery image + addRecoveryKeyword(recHeader); + } + // add keyword indicating that streaks have been removed + addDestreakKeyword(maskHeader); if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", @@ -373,10 +441,11 @@ streaksExit("", PS_EXIT_DATA_ERROR); } - // TODO: add keyword indicating that this is the recovery image - if (remove && sfiles->recMask && !psFitsWriteBlank(sfiles->recMask->fits, maskHeader, NULL)) { + if (recHeader && !psFitsWriteBlank(sfiles->recMask->fits, recHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", sfiles->recMask->resolved_name); streaksExit("", PS_EXIT_DATA_ERROR); } + psFree(recHeader); + recHeader = NULL; psFree(maskHeader); } @@ -389,5 +458,12 @@ streaksExit("", 1); } - // TODO: add keyword indicating that streaks have been removed + if (remove && sfiles->recWeight) { + recHeader = psMetadataCopy(NULL, weightHeader); + // add keyword indicating that this is a recovery image + addRecoveryKeyword(recHeader); + } + + // add keyword indicating that streaks have been removed + addDestreakKeyword(weightHeader); if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", @@ -395,6 +471,5 @@ streaksExit("", PS_EXIT_DATA_ERROR); } - // TODO: add keyword indicating that this is a recovery image - if (remove && sfiles->recWeight && !psFitsWriteBlank(sfiles->recWeight->fits, weightHeader, NULL)) { + if (recHeader && !psFitsWriteBlank(sfiles->recWeight->fits, recHeader, NULL)) { psError(PS_ERR_IO, false, "failed to write primary header to %s", sfiles->recWeight->resolved_name); @@ -402,4 +477,5 @@ } psFree(weightHeader); + psFree(recHeader); } } @@ -566,5 +642,6 @@ static void -setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero) +setFitsOptions(sFile *sfile, int bitpix, float bscale, float bzero, psFitsCompressionType compType, + psVector *tiles) { if (!sfile) { @@ -579,9 +656,17 @@ sfile->fits->options->bscale = bscale; sfile->fits->options->bzero = bzero; -} - -void -copyFitsOptions(sFile *out, sFile *rec, sFile *in) -{ + + psFitsSetCompression(sfile->fits, compType, tiles, 8, 0, 0); +} + +void +copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles) +{ + bool mdok; + psString compTypeStr = psMetadataLookupStr(&mdok, in->header, "ZCMPTYPE"); + psFitsCompressionType compType = psFitsCompressionTypeFromString(compTypeStr); + if (compType == PS_FITS_COMPRESS_NONE) { + return; + } // Get current BITPIX, BSCALE, BZERO, EXTNAME // Probably not necessary to look the numerical values up in this @@ -610,8 +695,7 @@ #ifdef STREAKS_COMPRESS_OUTPUT - // Paul says that I should be able to leave this blank - bitpix = 0; - setFitsOptions(out, bitpix, bscale, bzero); - setFitsOptions(rec, bitpix, bscale, bzero); + // printf("%d %f %f\n", bitpix, bscale, bzero); + setFitsOptions(out, bitpix, bscale, bzero, compType, tiles); + setFitsOptions(rec, bitpix, bscale, bzero, compType, tiles); #endif } @@ -790,19 +874,27 @@ bool -replicate(sFile *sfile, void *xattr) -{ - if (!sfile->inNebulous) { +replicate(sFile *outFile, sFile *inFile) +{ + if (!outFile->inNebulous) { return true; } nebServer *server = getNebServer(NULL); - // for now just set "user.copies" to 2 - if (!nebSetXattr(server, sfile->name, "user.copies", "2", NEB_REPLACE)) { - psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server)); + char *user_copies = nebGetXattr(server, inFile->name, "user.copies"); + bool free_user_copies = true; + if (user_copies == NULL) { + user_copies = "2"; + free_user_copies = false; + } + if (!nebSetXattr(server, outFile->name, "user.copies", user_copies, NEB_REPLACE)) { + psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", outFile->name, nebErr(server)); return false; } - if (!nebReplicate(server, sfile->name, NULL, NULL)) { - psError(PM_ERR_UNKNOWN, true, "nebSetXattr failed for %s\n%s", sfile->name, nebErr(server)); + if (!nebReplicate(server, outFile->name, "any", NULL)) { + psError(PM_ERR_UNKNOWN, true, "nebReplicate failed for %s\n%s", outFile->name, nebErr(server)); return false; + } + if (free_user_copies) { + nebFree(user_copies); } return true; @@ -817,35 +909,29 @@ bool status = false; - // XXX: TODO: need a nebGetXatrr function, but there isn't one - // another option would be to take the number of copies to be - // created as an option. That way the system could decide - // whether to replicate anything other than raw Image files - void *xattr = NULL; - - if (!replicate(sfiles->outImage, xattr)) { + if (!replicate(sfiles->outImage, sfiles->inImage)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; } -#ifdef notyet - // XXX: don't replicate mask and weight images until we can look up - // the input's xattr. There may be a perl program that can getXattr if (sfiles->outMask) { - // get xattr from input to see if we need to replicate - if (!replicate(sfiles->outMask, xattr)) { + if (!replicate(sfiles->outMask, sfiles->inMask)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; } } - if (sfiles->outWeight) { - // get xattr from input to see if we need to replicate - if (!replicate(sfiles->outWeight, xattr)) { + if (sfiles->outChMask) { + if (!replicate(sfiles->outChMask, sfiles->inChMask)) { psError(PM_ERR_SYS, false, "failed to replicate outImage."); return false; } } -#endif - -// replicate the recovery images (if in nebulous) + if (sfiles->outWeight) { + if (!replicate(sfiles->outWeight, sfiles->inWeight)) { + psError(PM_ERR_SYS, false, "failed to replicate outImage."); + return false; + } + } + +// XXX: replicate the recovery images (if in nebulous) // perhaps whether we do that or not should be configurable. // Sounds like we need a recipe @@ -904,4 +990,11 @@ } + if (sfiles->outChMask) { + if (!swapOutputToInput(sfiles->inChMask, sfiles->outChMask)) { + psError(PM_ERR_SYS, false, "failed to swap instances for chip mask."); + return false; + } + } + if (!swapOutputToInput(sfiles->inImage, sfiles->outImage)) { psError(PM_ERR_SYS, false, "failed to swap instances for Image."); @@ -936,20 +1029,17 @@ { if (sfiles->outMask) { - if (!deleteFile(sfiles->outMask)) { - psError(PM_ERR_SYS, false, "failed to delete Mask."); - return false; - } + deleteFile(sfiles->outMask); + } + + if (sfiles->outChMask) { + deleteFile(sfiles->outChMask); } if (sfiles->outWeight) { - if (!deleteFile(sfiles->outWeight)) { - psError(PM_ERR_SYS, false, "failed to delete Weight."); - return false; - } - } - - if (!deleteFile(sfiles->outImage)) { - psError(PM_ERR_SYS, false, "failed to delete Image."); - return false; + deleteFile(sfiles->outWeight); + } + + if (sfiles->outImage) { + deleteFile(sfiles->outImage); } Index: branches/eam_branches/20090522/magic/remove/src/streaksio.h =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksio.h (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksio.h (revision 24557) @@ -14,5 +14,5 @@ void copyPHU(streakFiles *sfiles, bool remove); void copyTable(sFile *out, sFile *in, int extnum); -void copyFitsOptions(sFile *out, sFile *rec, sFile *in); +void copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles); void setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll); void strkGetMaskValues(streakFiles *sfiles, psU32 *maskStreak, psU32 *maskMask); @@ -20,6 +20,9 @@ void writeImage(sFile *sfile, psString extname, int extnum); void writeImageCube(sFile *sfile, psArray *imagecube, psString extname, int extnum); -bool replicate(sFile *sfile, void *xattr); +bool replicate(sFile *outFile, sFile *inFile); void readImageFrom_pmFile(streakFiles *sf); + +void addDestreakKeyword(psMetadata *); +void addRecoveryKeyword(psMetadata *); bool streakFilesNextExtension(streakFiles *sf); Index: branches/eam_branches/20090522/magic/remove/src/streaksrelease.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksrelease.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksrelease.c (revision 24557) @@ -23,16 +23,7 @@ } - psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS"); - if (!status) { - psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n"); - return PS_EXIT_CONFIG_ERROR; - } - psU8 poorWarp = (double) psMetadataLookupU8(&status, masks, "POOR.WARP"); - if (!status) { - psError(PM_ERR_CONFIG, false, "failed to lookup mask value for POOR.WARP in recipes\n"); - return PS_EXIT_CONFIG_ERROR; - } - // we're setting pixels with any mask bits execpt POOR.WARP to NAN - psU8 maskMask = ~poorWarp; + // Values to set for masked pixels + psU32 maskStreak = 0; // for the image and weight (usually NAN, MAXINT for integer images) + psU32 maskMask = 0; // value looked up for MASK.STREAK // Does true work here? @@ -64,4 +55,9 @@ } + // now that we've read the input files, lookup the mask values that we read + if (maskStreak == 0) { + strkGetMaskValues(sfiles, &maskStreak, &maskMask); + } + setMaskedToNAN(sfiles, maskMask, true); @@ -77,41 +73,4 @@ printf("time to close images: %f\n", psTimerClear("CLOSE_IMAGES")); -#ifdef NOTYET - if (!replicateOutputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to replicate output files"); - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - - if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) { - // swap the instances for the input and output - // Note this is a database operation. No file I/O is performed - if (!swapOutputsToInputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to swap files"); - - // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that - // it has done and give a detailed report of what happened - - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - - if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { - // delete the temporary storage objects (which now points to the original image(s) - if (!deleteTemps(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); - // XXX: Now what? At this point the output files have been swapped, so we can't - // repeat the operation. - - // Returning error status here is problematic. The inputs have been streak removed - // but they're still lying around - // Maybe just print an error message and - // let other system tools clean up - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - } - } -#endif // REPLACE, REMOVE printf("time to run streaksrelease: %f\n", psTimerClear("STREAKSREMOVE")); @@ -275,17 +234,5 @@ // set up the compression parameters -#ifdef STREAKS_COMPRESS_OUTPUT - // compression of the image pixels is disabled for now. Some consortium members - // have problems reading them - copyFitsOptions(sf->outImage, sf->recImage, sf->inImage); - - // XXX: TODO: can we derive these values from the input header? - // psFitsCompressionGet(sf->inImage->image) gives compression none - // perhaps we should just use the definition of COMP_IMG in the configuration - psFitsSetCompression(sf->outImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - if (sf->recImage) { - psFitsSetCompression(sf->recImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - } -#endif + copyFitsOptions(sf->outImage, sf->recImage, sf->inImage, sf->tiles); if (sf->inMask) { @@ -306,19 +253,8 @@ } -#ifdef STREAKS_COMPRESS_OUTPUT - // XXX: see note above - copyFitsOptions(sf->outMask, sf->recMask, sf->inMask); - psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - if (sf->recMask) { - psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - } + copyFitsOptions(sf->outMask, sf->recMask, sf->inMask, sf->tiles); if (sf->outChMask) { - copyFitsOptions(sf->outChMask, sf->recChMask, sf->inMask); - psFitsSetCompression(sf->outChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - if (sf->recChMask) { - psFitsSetCompression(sf->recChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - } - } -#endif + copyFitsOptions(sf->outChMask, sf->recChMask, sf->inMask, sf->tiles); + } } } @@ -332,12 +268,5 @@ setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll); -#ifdef STREAKS_COMPRESS_OUTPUT - copyFitsOptions(sf->outWeight, sf->recWeight, sf->inWeight); - // XXX: see note above - psFitsSetCompression(sf->outWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - if (sf->recWeight) { - psFitsSetCompression(sf->recWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - } -#endif + copyFitsOptions(sf->outWeight, sf->recWeight, sf->inWeight, sf->tiles); } // and for raw images, create sub images that represent the actual image Index: branches/eam_branches/20090522/magic/remove/src/streaksremove.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksremove.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksremove.c (revision 24557) @@ -1,2 +1,11 @@ +/* + * streaksremove + * + * Convert satellite streak detctions into masks and remove the covered pixels from the + * input images. + * Optionally swap the inputs with the outputs so that subsequent references to the original + * images access the destreaked versions. + */ + #include "streaksremove.h" @@ -23,18 +32,24 @@ } - psU32 maskStreak = 0; - psU32 maskMask = 0; + // Values to set for masked pixels + psU32 maskStreak = 0; // for the image and weight (usually NAN, MAXINT for integer images) + psU32 maskMask = 0; // value looked up for MASK.STREAK psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS"); + // call Paul Sydney's code to parse the streaks file that DetectStreaks produced Streaks *streaks = readStreaksFile(streaksFileName); if (!streaks) { - psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName); + psError(PS_ERR_UNKNOWN, "failed to read streaks file: %s", streaksFileName); streaksExit("", PS_EXIT_PROG_ERROR); } + // open all of the input and output files, save their descriptions in the streakFiles struct streakFiles *sfiles = openFiles(config, true, argv[0]); setupAstrometry(sfiles); + // Optionally we can set pixels that are masked to NAN since they couldn't have been + // examined for streaks. Usually this is done by the distribution system just prior + // to release bool nanForRelease = psMetadataLookupBool(&status, config->arguments, "NAN_FOR_RELEASE"); if (nanForRelease && (sfiles->inMask == NULL)) { @@ -52,5 +67,5 @@ if (checkNonWarpedPixels ) { - // From ICD: + // From magic ICD: // In the raw and detrended images, the pixels which were not // included in any of the streak-processed warps must also be masked. @@ -69,5 +84,7 @@ if (sfiles->stage == IPP_STAGE_RAW) { - // copy PHU to output files + // Except for raw stage, all of our (GPC1) files have one image extension. + // Raw files have a phu and multiple extensions, one per chip + // Since this is a raw file, copy it's PHU to output files copyPHU(sfiles, true); @@ -82,5 +99,5 @@ int totalStreakPixels = 0; - // Iterate through each component of the input (there is only one except for raw images) + // Iterate through each component of the input (except for raw images there is only one) do { bool exciseImageCube = false; @@ -110,10 +127,11 @@ psTimerStart("GET_STREAK_PIXELS"); - StreakPixels *pixels = streak_on_component (streaks, sfiles->astrom, - sfiles->inImage->numCols, sfiles->inImage->numRows); - + // call Paul Sydney's code to compute the set of pixels that are covered by the detected streaks + StreakPixels *pixels = streak_on_component(streaks, sfiles->astrom, sfiles->inImage->numCols, + sfiles->inImage->numRows); psLogMsg("streaksremove", PS_LOG_INFO, "time to get streak pixels: %f\n", psTimerClear("GET_STREAK_PIXELS")); - + // if this extension contained an image, excise the streaked pixels. + // otherwise it contained an image cube (video cell) which is handled in the if block if (sfiles->inImage->image) { if (checkNonWarpedPixels) { @@ -126,9 +144,15 @@ psLogMsg("streaksremove", PS_LOG_INFO, "time to excise non warped pixels: %f\n", psTimerClear("EXCISE_NON_WARPED")); } + totalStreakPixels += psArrayLength(pixels); + psTimerStart("REMOVE_STREAKS"); + + // for each pixel covered by the streak for (int i = 0; i < psArrayLength (pixels); ++i) { PixelPos *pixelPos = psArrayGet (pixels, i); + // if this pixel was not part of the warp, skip because it has already been + // excised (unless we weren't asked to check) if (!checkNonWarpedPixels || warpedPixel(sfiles, pixelPos)) { @@ -140,12 +164,15 @@ } } + psLogMsg("streaksremove", PS_LOG_INFO, "time to remove streak pixels: %f\n", psTimerClear("REMOVE_STREAKS")); if (nanForRelease) { + // set any pixels that were masked, to NAN (unless they are already NAN) setMaskedToNAN(sfiles, maskMask, true); } } else { - // this component contains an image cube, excise it completely + // this component contains an image cube + // For now excise it completely exciseImageCube = true; } @@ -155,8 +182,11 @@ if (sfiles->stage == IPP_STAGE_CHIP) { + // as a convience to the user of the output, replace the bogus WCS transform in the + // chip processed files with the data calcuated by psastro at the camera stage + // (actually we use a linear approximation) updateAstrometry(sfiles); } - // write out the destreaked temporary images and the recovery images + // write the destreaked "temporary" images and the recovery images writeImages(sfiles, exciseImageCube); @@ -168,47 +198,40 @@ psLogMsg("streaksremove", PS_LOG_INFO, "pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, 100. * totalStreakPixels / totalPixels); + // all done close the files. This is where the files are written so it can take a long time. + psTimerStart("CLOSE_IMAGES"); - // close all files + closeImages(sfiles); + psLogMsg("streaksremove", PS_LOG_INFO, "time to close images: %f\n", psTimerClear("CLOSE_IMAGES")); + + + if (!replicateOutputs(sfiles)) { + psError(PS_ERR_UNKNOWN, false, "failed to replicate output files"); + deleteTemps(sfiles); + psErrorStackPrint(stderr, ""); + exit(PS_EXIT_UNKNOWN_ERROR); + } // NOTE: from here on we can't just quit if something goes wrong. // especially if we're working at the raw stage - - if (!replicateOutputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to replicate output files"); - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } + // turn off automatic deletion of output files by streaksExit + setStreakFiles(NULL); if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) { // swap the instances for the input and output - // Note this is a database operation. No file I/O is performed + // Note this is a nebulous database operation. No file I/O is performed if (!swapOutputsToInputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to swap files"); - - // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that - // it has done and give a detailed report of what happened - - psErrorStackPrint(stderr, ""); + // XXX: Now what? + // It is up to the program that reverts failed destreak runs to insure that + // any input files that have been swapped are restored + + psErrorStackPrint(stderr, "failed to swap files"); + + // XXX: pick a specific error code for this failure exit(PS_EXIT_UNKNOWN_ERROR); } - - if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { - // delete the temporary storage objects (which now points to the original image(s) - if (!deleteTemps(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); - // XXX: Now what? At this point the output files have been swapped, so we can't - // repeat the operation. - - // Returning error status here is problematic. The inputs have been streak removed - // but they're still lying around - // Maybe just print an error message and - // let other system tools clean up - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - } - } + } + // all done. Clean up to look for memory leaks. psFree(sfiles); @@ -403,8 +426,6 @@ usage(); } - psString dir = pathToDirectory(argv[argnum]); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files", - dir); - psFree(dir); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "path for (temporary if replae) output files", + argv[argnum]); psArgumentRemove(argnum, &argc, argv); } else { @@ -415,22 +436,10 @@ if ((argnum = psArgumentGet(argc, argv, "-recovery"))) { psArgumentRemove(argnum, &argc, argv); - psString dir = pathToDirectory(argv[argnum]); - psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files", - dir); - psFree(dir); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "path for recovery files", + argv[argnum]); psArgumentRemove(argnum, &argc, argv); } else if ((stage == IPP_STAGE_RAW) && gotReplace) { psError(PS_ERR_UNKNOWN, true, "-recovery is required for -stage raw with -replace\n"); usage(); - } - - if ((argnum = psArgumentGet(argc, argv, "-remove"))) { - if (!gotReplace) { - psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n"); - usage(); - } - psArgumentRemove(argnum, &argc, argv); - psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files", - true); } @@ -451,4 +460,5 @@ updateAstrometry(streakFiles *sf) { + // XXX: why do I check this here? Shouldn't it be just around the call to linearizeTransforms? if (sf->bilevelAstrometry) { @@ -501,8 +511,10 @@ } } - sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header); + sf->outImage->header = psMemIncrRefCounter(sf->inImage->header); if (sf->recImage) { - sf->recImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header); - } + sf->recImage->header = psMetadataCopy(NULL, sf->inImage->header); + addRecoveryKeyword(sf->recImage->header); + } + addDestreakKeyword(sf->outImage->header); if (!SFILE_IS_IMAGE(sf->inImage)) { @@ -520,17 +532,5 @@ // set up the compression parameters -#ifdef STREAKS_COMPRESS_OUTPUT - // compression of the image pixels is disabled for now. Some consortium members - // have problems reading them - copyFitsOptions(sf->outImage, sf->recImage, sf->inImage); - - // XXX: TODO: can we derive these values from the input header? - // psFitsCompressionGet(sf->inImage->image) gives compression none - // perhaps we should just use the definition of COMP_IMG in the configuration - psFitsSetCompression(sf->outImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - if (sf->recImage) { - psFitsSetCompression(sf->recImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - } -#endif + copyFitsOptions(sf->outImage, sf->recImage, sf->inImage, sf->tiles); if (sf->inMask) { @@ -539,6 +539,8 @@ sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header); if (sf->recMask) { - sf->recMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header); - } + sf->recMask->header = psMetadataCopy(NULL, sf->outMask->header); + addRecoveryKeyword(sf->recMask->header); + } + addDestreakKeyword(sf->outMask->header); if (updateAstrometry) { pmAstromWriteWCS(sf->outMask->header, sf->inAstrom->fpa, sf->chip, 0.001); @@ -554,19 +556,8 @@ } -#ifdef STREAKS_COMPRESS_OUTPUT - // XXX: see note above - copyFitsOptions(sf->outMask, sf->recMask, sf->inMask); - psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - if (sf->recMask) { - psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - } + copyFitsOptions(sf->outMask, sf->recMask, sf->inMask, sf->tiles); if (sf->outChMask) { - copyFitsOptions(sf->outChMask, sf->recChMask, sf->inMask); - psFitsSetCompression(sf->outChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - if (sf->recChMask) { - psFitsSetCompression(sf->recChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); - } - } -#endif + copyFitsOptions(sf->outChMask, sf->recChMask, sf->inMask, sf->tiles); + } } } @@ -576,6 +567,8 @@ sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header); if (sf->recWeight) { - sf->recWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header); - } + sf->recWeight->header = psMetadataCopy(NULL, sf->outWeight->header); + addRecoveryKeyword(sf->recWeight->header); + } + addDestreakKeyword(sf->outWeight->header); if (updateAstrometry) { pmAstromWriteWCS(sf->inWeight->header, sf->inAstrom->fpa, sf->chip, 0.001); @@ -583,20 +576,9 @@ setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll); -#ifdef STREAKS_COMPRESS_OUTPUT - copyFitsOptions(sf->outWeight, sf->recWeight, sf->inWeight); - // XXX: see note above - psFitsSetCompression(sf->outWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - if (sf->recWeight) { - psFitsSetCompression(sf->recWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); - } -#endif - } - // and for raw images, create sub images that represent the actual image - // area (no overscan) + copyFitsOptions(sf->outWeight, sf->recWeight, sf->inWeight, sf->tiles); + } return true; } - - static void Index: branches/eam_branches/20090522/magic/remove/src/streaksremove.h =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksremove.h (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksremove.h (revision 24557) @@ -88,4 +88,5 @@ extern ippStage parseStage(psString); extern psString pathToDirectory(char *path); +extern void setStreakFiles( streakFiles *); #define CHIP_LEVEL_INPUT(_stage) ((_stage == IPP_STAGE_RAW) || (_stage == IPP_STAGE_CHIP)) @@ -95,3 +96,4 @@ #define IN_NEBULOUS(_filename) (!strncasecmp(_filename, "neb://", strlen("neb://"))) + #endif // STREAKS_H Index: branches/eam_branches/20090522/magic/remove/src/streaksreplace.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksreplace.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksreplace.c (revision 24557) @@ -97,35 +97,4 @@ } -#ifdef NOTYET - if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) { - // swap the instances for the input and output - // Note this is a database operation. No file I/O is performed - if (!swapOutputsToInputs(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to swap files"); - - // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that - // it has done and give a detailed report of what happened - - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - - if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) { - // delete the temporary storage objects (which now points to the original image(s) - if (!deleteTemps(sfiles)) { - psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files"); - // XXX: Now what? At this point the output files have been swapped, so we can't - // repeat the operation. - - // Returning error status here is problematic. The inputs have been streak removed - // but they're still lying around - // Maybe just print an error message and - // let other system tools clean up - psErrorStackPrint(stderr, ""); - exit(PS_EXIT_UNKNOWN_ERROR); - } - } - } -#endif // REPLACE, REMOVE // nebServerFree(ourNebServer); psFree(config); @@ -344,14 +313,6 @@ // set up the compression parameters -#ifdef STREAKS_COMPRESS_OUTPUT - // compression of the image pixels is disabled for now. Some consortium members - // have problems reading compressed images. - copyFitsOptions(sf->outImage, sf->recImage, sf->inImage); - - // XXX: TODO: can we derive these values from the input header? - // psFitsCompressionGet(sf->inImage->image) gives compression none - // perhaps we should just use the definition of COMP_IMG in the configuration - psFitsSetCompression(sf->outImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); -#endif + copyFitsOptions(sf->outImage, sf->recImage, sf->inImage, sf->tiles); + if (sf->inMask) { readImage(sf->inMask, sf->extnum, sf->stage, true); @@ -362,7 +323,5 @@ setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false); - // XXX: see note above - copyFitsOptions(sf->outMask, NULL, sf->inMask); - psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0); + copyFitsOptions(sf->outMask, NULL, sf->inMask, sf->tiles); } @@ -374,7 +333,5 @@ setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false); - copyFitsOptions(sf->outWeight, NULL, sf->inWeight); - // XXX: see note above - psFitsSetCompression(sf->outWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0); + copyFitsOptions(sf->outWeight, NULL, sf->inWeight, sf->tiles); } Index: branches/eam_branches/20090522/magic/remove/src/streaksutil.c =================================================================== --- branches/eam_branches/20090522/magic/remove/src/streaksutil.c (revision 24529) +++ branches/eam_branches/20090522/magic/remove/src/streaksutil.c (revision 24557) @@ -33,8 +33,19 @@ } +streakFiles *ourStreakFiles = NULL; + +void +setStreakFiles(streakFiles *sfiles) +{ + ourStreakFiles = sfiles; +} + // to enhance clarity in these programs we don't propagate errors up the stack // we just bail out void streaksExit(psString str, int exitCode) { psErrorStackPrint(stderr, str); + if (ourStreakFiles) { + deleteTemps(ourStreakFiles); + } exit(exitCode); } Index: branches/eam_branches/20090522/portage/bin/post_sync =================================================================== --- branches/eam_branches/20090522/portage/bin/post_sync (revision 24557) +++ branches/eam_branches/20090522/portage/bin/post_sync (revision 24557) @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright 2006-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +if [ -d /etc/portage/postsync.d/ ]; then + for f in /etc/portage/postsync.d/* ; do + if [ -x ${f} ] ; then + ${f} + fi + done +else + : +fi Index: branches/eam_branches/20090522/portage/categories =================================================================== --- branches/eam_branches/20090522/portage/categories (revision 24557) +++ branches/eam_branches/20090522/portage/categories (revision 24557) @@ -0,0 +1,1 @@ +perl-gcpan Index: branches/eam_branches/20090522/portage/make.conf =================================================================== --- branches/eam_branches/20090522/portage/make.conf (revision 24557) +++ branches/eam_branches/20090522/portage/make.conf (revision 24557) @@ -0,0 +1,34 @@ +# These settings were set by the catalyst build script that automatically built this stage +# Please consult /etc/make.conf.example for a more detailed example +CFLAGS="-O2 -pipe -march=k8" +CHOST="x86_64-pc-linux-gnu" +CXXFLAGS="${CFLAGS}" +ACCEPT_KEYWORDS="amd64" +MAKEOPTS="-j9" +# build box only features +FEATURES="buildpkg ccache parallel-fetch fixpackages assume-digests userfetch userpriv usersandbox sfperms suidctl" +#FEATURES="ccache parallel-fetch userfetch userpriv usersandbox sfperms suidctl" +CCACHE_SIZE=8G +PORTAGE_NICENESS=10 +#CONFIG_PROTECT=/usr/share/genkernel +CONFIG_PROTECT_MASK="/etc/init.d /usr/share /etc/X11 /etc/fonts /etc/mono /etc/mutt /etc/xdg /etc/dbus-1 /etc/gimp /etc/udev" +CLEAN_DELAY=2 +USE="X xinerama gnome gnome2 gtk gtk2 cairo vim-with-x xv imap mbox \ + firefox cups samba apache2 perl fortran gif jpeg jbig png xpm \ + jpeg2k graphviz tiff mpeg lcms svg truetype fontconfig acpi sysfs \ + pic unicode pcre bash-completion acl xattr nfs kpathsea nptl \ + -nptlonly -java -qt -kde openmp hal subversion" +INPUT_DEVICES="keyboard mouse evdev" +VIDEO_CARDS="r128 radeon nv vga" +#SYNC=rsync://gentoo.ifa.hawaii.edu/gentoo-portage-ipp +SYNC=rsync://gentoo.ifa.hawaii.edu/gentoo-portage-ipp +PORTDIR="/data/portage" +PORTDIR_OVERLAY="/data/portage_overlay" +# keep distfiles on the local disk so /data/portage can be mounted ro +DISTDIR="/portage/distfiles" +PKGDIR="/data/portage/packages" +#http_proxy="gentoo.ifa.hawaii.edu:8888" +APACHE2_MODULES="actions alias asis auth_digest autoindex cache dav deflate dir env expires headers include info log_config logio mime negotiation proxy rewrite setenvif speling status userdir usertrack auth_basic authn_alias authn_anon authn_dbd authn_dbm authn_default authn_file authz_dbm authz_default authz_groupfile authz_host authz_owner authz_user cern_meta charset_lite dav_fs dav_lock dbd disk_cache dumpio ext_filter file_cache filter ident imagemap log_forensic mem_cache mime_magic proxy_ajp proxy_balancer proxy_connect proxy_ftp proxy_http unique_id version vhost_alias" +APACHE2_MPMS=prefork +#GENTOO_MIRRORS="http://ipp0002.ifa.hawaii.edu/~jhoblitt/" +PORTAGE_WORKDIR_MODE=0700 Index: branches/eam_branches/20090522/portage/package.keywords =================================================================== --- branches/eam_branches/20090522/portage/package.keywords (revision 24557) +++ branches/eam_branches/20090522/portage/package.keywords (revision 24557) @@ -0,0 +1,33 @@ +sys-cluster/ganglia ~amd64 +x11-terms/pssh ~amd64 +sys-cluster/pvm ~amd64 +dev-perl/JSON ~amd64 +dev-util/svk ~amd64 +dev-util/subversion ~amd64 +dev-util/git ~amd64 +sci-libs/pgplot ~amd64 +sys-kernel/gentoo-sources ~amd64 +sys-power/pmtools ~amd64 +www-apache/pwauth ~amd64 +dev-perl/Filesys-Df x86 +www-apache/mod_perl ~amd64 +dev-python/setuptools ~amd64 +dev-db/mysql-community ~amd64 +x11-terms/pssh ~amd64 +dev-perl/Params-Validate ~amd64 +dev-perl/yaml ~amd64 +dev-perl/App-CLI ~amd64 +dev-perl/DateTime ~amd64 +dev-perl/DateTime-TimeZone ~amd64 +dev-perl/DateTime-Format-Builder ~amd64 +dev-perl/DateTime-Format-Strptime ~amd64 +dev-perl/DateTime-Format-ISO8601 ~amd64 +virtual/perl-Module-Load-Conditional ~amd64 +perl-core/Module-Load-Conditional ~amd64 +virtual/perl-Params-Check ~amd64 +perl-core/Params-Check ~amd64 +virtual/perl-Module-Load ~amd64 +perl-core/Module-Load ~amd64 +perl-core/IPC-Cmd ~amd64 +virtual/perl-IPC-Cmd ~amd64 +dev-perl/Sys-Statistics-Linux ~amd64 #0.34 ebuild broken Index: branches/eam_branches/20090522/portage/package.mask =================================================================== --- branches/eam_branches/20090522/portage/package.mask (revision 24557) +++ branches/eam_branches/20090522/portage/package.mask (revision 24557) @@ -0,0 +1,4 @@ +>sys-boot/grub-0.97-r3 +>sys-kernel/genkernel-3.4.10-r1 +>=sys-apps/archttp-1.82 +=app-shells/bash-completion-20050121-r10 Index: branches/eam_branches/20090522/portage/package.use =================================================================== --- branches/eam_branches/20090522/portage/package.use (revision 24557) +++ branches/eam_branches/20090522/portage/package.use (revision 24557) @@ -0,0 +1,4 @@ +dev-lang/php apache2 berkdb cli crypt gdbm iconv imap ipv6 ncurses nls pcre pic readline reflection session spl ssl truetype unicode xpm zlib cgi exif gd gmp hash inifile json mysql soap sockets xml xmlreader xmlrpc xmlwriter xsl ctype +www-apps/trac cgi fastcgi mysql sqlite subversion vhosts +www-servers/apache suexec +dev-util/subversion -dso Index: branches/eam_branches/20090522/portage/postsync.d/q-reinitialize =================================================================== --- branches/eam_branches/20090522/portage/postsync.d/q-reinitialize (revision 24557) +++ branches/eam_branches/20090522/portage/postsync.d/q-reinitialize (revision 24557) @@ -0,0 +1,1 @@ +[ -x /usr/bin/q ] && /usr/bin/q -qr Index: branches/eam_branches/20090522/portage/savedconfig/sys-apps/busybox-1.8.2 =================================================================== --- branches/eam_branches/20090522/portage/savedconfig/sys-apps/busybox-1.8.2 (revision 24557) +++ branches/eam_branches/20090522/portage/savedconfig/sys-apps/busybox-1.8.2 (revision 24557) @@ -0,0 +1,771 @@ +# +# Automatically generated make config: don't edit +# Busybox version: 1.8.2 +# Thu Jun 26 16:29:34 2008 +# +CONFIG_HAVE_DOT_CONFIG=y + +# +# Busybox Settings +# + +# +# General Configuration +# +CONFIG_NITPICK=y +CONFIG_DESKTOP=y +CONFIG_FEATURE_BUFFERS_USE_MALLOC=y +# CONFIG_FEATURE_BUFFERS_GO_ON_STACK is not set +# CONFIG_FEATURE_BUFFERS_GO_IN_BSS is not set +CONFIG_SHOW_USAGE=y +CONFIG_FEATURE_VERBOSE_USAGE=y +CONFIG_FEATURE_COMPRESS_USAGE=y +CONFIG_FEATURE_INSTALLER=y +# CONFIG_LOCALE_SUPPORT is not set +CONFIG_GETOPT_LONG=y +CONFIG_FEATURE_DEVPTS=y +CONFIG_FEATURE_CLEAN_UP=y +CONFIG_FEATURE_PIDFILE=y +CONFIG_FEATURE_SUID=y +# CONFIG_FEATURE_SUID_CONFIG is not set +# CONFIG_FEATURE_SUID_CONFIG_QUIET is not set +# CONFIG_SELINUX is not set +CONFIG_FEATURE_PREFER_APPLETS=y +CONFIG_BUSYBOX_EXEC_PATH="/proc/self/exe" +CONFIG_FEATURE_SYSLOG=y +CONFIG_FEATURE_HAVE_RPC=y + +# +# Build Options +# +# CONFIG_STATIC is not set +# CONFIG_BUILD_LIBBUSYBOX is not set +# CONFIG_FEATURE_INDIVIDUAL is not set +# CONFIG_FEATURE_SHARED_BUSYBOX is not set +CONFIG_LFS=y + +# +# Debugging Options +# +# CONFIG_DEBUG is not set +CONFIG_WERROR=y +CONFIG_NO_DEBUG_LIB=y +# CONFIG_DMALLOC is not set +# CONFIG_EFENCE is not set +CONFIG_INCLUDE_SUSv2=y + +# +# Installation Options +# +CONFIG_INSTALL_NO_USR=y +CONFIG_INSTALL_APPLET_SYMLINKS=y +# CONFIG_INSTALL_APPLET_HARDLINKS is not set +# CONFIG_INSTALL_APPLET_SCRIPT_WRAPPERS is not set +# CONFIG_INSTALL_APPLET_DONT is not set +# CONFIG_INSTALL_SH_APPLET_SYMLINK is not set +# CONFIG_INSTALL_SH_APPLET_HARDLINK is not set +# CONFIG_INSTALL_SH_APPLET_SCRIPT_WRAPPER is not set +CONFIG_PREFIX="./_install" + +# +# Busybox Library Tuning +# +CONFIG_PASSWORD_MINLEN=6 +CONFIG_MD5_SIZE_VS_SPEED=2 +CONFIG_FEATURE_FAST_TOP=y +CONFIG_FEATURE_ETC_NETWORKS=y +CONFIG_FEATURE_EDITING=y +CONFIG_FEATURE_EDITING_MAX_LEN=1024 +CONFIG_FEATURE_EDITING_FANCY_KEYS=y +CONFIG_FEATURE_EDITING_VI=y +CONFIG_FEATURE_EDITING_HISTORY=15 +CONFIG_FEATURE_EDITING_SAVEHISTORY=y +CONFIG_FEATURE_TAB_COMPLETION=y +CONFIG_FEATURE_USERNAME_COMPLETION=y +CONFIG_FEATURE_EDITING_FANCY_PROMPT=y +CONFIG_MONOTONIC_SYSCALL=y +CONFIG_IOCTL_HEX2STR_ERROR=y + +# +# Applets +# + +# +# Archival Utilities +# +CONFIG_AR=y +CONFIG_FEATURE_AR_LONG_FILENAMES=y +CONFIG_BUNZIP2=y +CONFIG_BZIP2=y +CONFIG_CPIO=y +# CONFIG_DPKG is not set +CONFIG_DPKG_DEB=y +CONFIG_FEATURE_DPKG_DEB_EXTRACT_ONLY=y +CONFIG_GUNZIP=y +CONFIG_FEATURE_GUNZIP_UNCOMPRESS=y +CONFIG_GZIP=y +# CONFIG_RPM2CPIO is not set +# CONFIG_RPM is not set +# CONFIG_FEATURE_RPM_BZ2 is not set +CONFIG_TAR=y +CONFIG_FEATURE_TAR_CREATE=y +CONFIG_FEATURE_TAR_BZIP2=y +CONFIG_FEATURE_TAR_LZMA=y +CONFIG_FEATURE_TAR_FROM=y +CONFIG_FEATURE_TAR_GZIP=y +CONFIG_FEATURE_TAR_COMPRESS=y +CONFIG_FEATURE_TAR_OLDGNU_COMPATIBILITY=y +CONFIG_FEATURE_TAR_OLDSUN_COMPATIBILITY=y +CONFIG_FEATURE_TAR_GNU_EXTENSIONS=y +CONFIG_FEATURE_TAR_LONG_OPTIONS=y +CONFIG_UNCOMPRESS=y +CONFIG_UNLZMA=y +CONFIG_FEATURE_LZMA_FAST=y +CONFIG_UNZIP=y + +# +# Common options for cpio and tar +# +CONFIG_FEATURE_UNARCHIVE_TAPE=y + +# +# Common options for dpkg and dpkg_deb +# +CONFIG_FEATURE_DEB_TAR_GZ=y +CONFIG_FEATURE_DEB_TAR_BZ2=y +CONFIG_FEATURE_DEB_TAR_LZMA=y + +# +# Coreutils +# +CONFIG_BASENAME=y +CONFIG_CAL=y +CONFIG_CAT=y +CONFIG_CATV=y +CONFIG_CHGRP=y +CONFIG_CHMOD=y +CONFIG_CHOWN=y +CONFIG_CHROOT=y +CONFIG_CKSUM=y +CONFIG_COMM=y +CONFIG_CP=y +CONFIG_CUT=y +CONFIG_DATE=y +CONFIG_FEATURE_DATE_ISOFMT=y +CONFIG_DD=y +CONFIG_FEATURE_DD_SIGNAL_HANDLING=y +CONFIG_FEATURE_DD_IBS_OBS=y +CONFIG_DF=y +CONFIG_DIRNAME=y +CONFIG_DOS2UNIX=y +CONFIG_UNIX2DOS=y +CONFIG_DU=y +CONFIG_FEATURE_DU_DEFAULT_BLOCKSIZE_1K=y +CONFIG_ECHO=y +CONFIG_FEATURE_FANCY_ECHO=y +CONFIG_ENV=y +CONFIG_FEATURE_ENV_LONG_OPTIONS=y +CONFIG_EXPAND=y +CONFIG_FEATURE_EXPAND_LONG_OPTIONS=y +CONFIG_EXPR=y +CONFIG_EXPR_MATH_SUPPORT_64=y +CONFIG_FALSE=y +# CONFIG_FOLD is not set +CONFIG_HEAD=y +CONFIG_FEATURE_FANCY_HEAD=y +CONFIG_HOSTID=y +CONFIG_ID=y +CONFIG_INSTALL=y +CONFIG_FEATURE_INSTALL_LONG_OPTIONS=y +CONFIG_LENGTH=y +CONFIG_LN=y +# CONFIG_LOGNAME is not set +CONFIG_LS=y +CONFIG_FEATURE_LS_FILETYPES=y +CONFIG_FEATURE_LS_FOLLOWLINKS=y +CONFIG_FEATURE_LS_RECURSIVE=y +CONFIG_FEATURE_LS_SORTFILES=y +CONFIG_FEATURE_LS_TIMESTAMPS=y +CONFIG_FEATURE_LS_USERNAME=y +CONFIG_FEATURE_LS_COLOR=y +CONFIG_FEATURE_LS_COLOR_IS_DEFAULT=y +CONFIG_MD5SUM=y +CONFIG_MKDIR=y +CONFIG_FEATURE_MKDIR_LONG_OPTIONS=y +CONFIG_MKFIFO=y +CONFIG_MKNOD=y +CONFIG_MV=y +CONFIG_FEATURE_MV_LONG_OPTIONS=y +CONFIG_NICE=y +CONFIG_NOHUP=y +# CONFIG_OD is not set +CONFIG_PRINTENV=y +CONFIG_PRINTF=y +CONFIG_PWD=y +CONFIG_READLINK=y +CONFIG_FEATURE_READLINK_FOLLOW=y +CONFIG_REALPATH=y +CONFIG_RM=y +CONFIG_RMDIR=y +CONFIG_SEQ=y +CONFIG_SHA1SUM=y +CONFIG_SLEEP=y +CONFIG_FEATURE_FANCY_SLEEP=y +CONFIG_SORT=y +CONFIG_FEATURE_SORT_BIG=y +CONFIG_SPLIT=y +CONFIG_FEATURE_SPLIT_FANCY=y +CONFIG_STAT=y +CONFIG_FEATURE_STAT_FORMAT=y +CONFIG_STTY=y +CONFIG_SUM=y +CONFIG_SYNC=y +CONFIG_TAIL=y +CONFIG_FEATURE_FANCY_TAIL=y +CONFIG_TEE=y +CONFIG_FEATURE_TEE_USE_BLOCK_IO=y +CONFIG_TEST=y +CONFIG_FEATURE_TEST_64=y +CONFIG_TOUCH=y +CONFIG_TR=y +CONFIG_FEATURE_TR_CLASSES=y +CONFIG_FEATURE_TR_EQUIV=y +CONFIG_TRUE=y +CONFIG_TTY=y +CONFIG_UNAME=y +CONFIG_UNEXPAND=y +CONFIG_FEATURE_UNEXPAND_LONG_OPTIONS=y +CONFIG_UNIQ=y +CONFIG_USLEEP=y +# CONFIG_UUDECODE is not set +# CONFIG_UUENCODE is not set +CONFIG_WC=y +CONFIG_FEATURE_WC_LARGE=y +CONFIG_WHO=y +CONFIG_WHOAMI=y +CONFIG_YES=y + +# +# Common options for cp and mv +# +CONFIG_FEATURE_PRESERVE_HARDLINKS=y + +# +# Common options for ls, more and telnet +# +CONFIG_FEATURE_AUTOWIDTH=y + +# +# Common options for df, du, ls +# +CONFIG_FEATURE_HUMAN_READABLE=y + +# +# Common options for md5sum, sha1sum +# +CONFIG_FEATURE_MD5_SHA1_SUM_CHECK=y + +# +# Console Utilities +# +CONFIG_CHVT=y +CONFIG_CLEAR=y +CONFIG_DEALLOCVT=y +CONFIG_DUMPKMAP=y +CONFIG_KBD_MODE=y +CONFIG_LOADFONT=y +CONFIG_LOADKMAP=y +CONFIG_OPENVT=y +CONFIG_RESET=y +CONFIG_RESIZE=y +CONFIG_FEATURE_RESIZE_PRINT=y +CONFIG_SETCONSOLE=y +CONFIG_FEATURE_SETCONSOLE_LONG_OPTIONS=y +CONFIG_SETKEYCODES=y +CONFIG_SETLOGCONS=y + +# +# Debian Utilities +# +CONFIG_MKTEMP=y +CONFIG_PIPE_PROGRESS=y +CONFIG_RUN_PARTS=y +CONFIG_FEATURE_RUN_PARTS_LONG_OPTIONS=y +CONFIG_FEATURE_RUN_PARTS_FANCY=y +CONFIG_START_STOP_DAEMON=y +CONFIG_FEATURE_START_STOP_DAEMON_FANCY=y +CONFIG_FEATURE_START_STOP_DAEMON_LONG_OPTIONS=y +CONFIG_WHICH=y + +# +# Editors +# +CONFIG_AWK=y +CONFIG_FEATURE_AWK_MATH=y +CONFIG_CMP=y +CONFIG_DIFF=y +CONFIG_FEATURE_DIFF_BINARY=y +CONFIG_FEATURE_DIFF_DIR=y +CONFIG_FEATURE_DIFF_MINIMAL=y +CONFIG_ED=y +CONFIG_PATCH=y +CONFIG_SED=y +CONFIG_VI=y +CONFIG_FEATURE_VI_MAX_LEN=1024 +CONFIG_FEATURE_VI_COLON=y +CONFIG_FEATURE_VI_YANKMARK=y +CONFIG_FEATURE_VI_SEARCH=y +CONFIG_FEATURE_VI_USE_SIGNALS=y +CONFIG_FEATURE_VI_DOT_CMD=y +CONFIG_FEATURE_VI_READONLY=y +CONFIG_FEATURE_VI_SETOPTS=y +CONFIG_FEATURE_VI_SET=y +CONFIG_FEATURE_VI_WIN_RESIZE=y +CONFIG_FEATURE_VI_OPTIMIZE_CURSOR=y +CONFIG_FEATURE_ALLOW_EXEC=y + +# +# Finding Utilities +# +CONFIG_FIND=y +CONFIG_FEATURE_FIND_PRINT0=y +CONFIG_FEATURE_FIND_MTIME=y +CONFIG_FEATURE_FIND_MMIN=y +CONFIG_FEATURE_FIND_PERM=y +CONFIG_FEATURE_FIND_TYPE=y +CONFIG_FEATURE_FIND_XDEV=y +CONFIG_FEATURE_FIND_MAXDEPTH=y +CONFIG_FEATURE_FIND_NEWER=y +CONFIG_FEATURE_FIND_INUM=y +CONFIG_FEATURE_FIND_EXEC=y +CONFIG_FEATURE_FIND_USER=y +CONFIG_FEATURE_FIND_GROUP=y +CONFIG_FEATURE_FIND_NOT=y +CONFIG_FEATURE_FIND_DEPTH=y +CONFIG_FEATURE_FIND_PAREN=y +CONFIG_FEATURE_FIND_SIZE=y +CONFIG_FEATURE_FIND_PRUNE=y +CONFIG_FEATURE_FIND_DELETE=y +CONFIG_FEATURE_FIND_PATH=y +CONFIG_FEATURE_FIND_REGEX=y +# CONFIG_FEATURE_FIND_CONTEXT is not set +CONFIG_GREP=y +CONFIG_FEATURE_GREP_EGREP_ALIAS=y +CONFIG_FEATURE_GREP_FGREP_ALIAS=y +CONFIG_FEATURE_GREP_CONTEXT=y +CONFIG_XARGS=y +CONFIG_FEATURE_XARGS_SUPPORT_CONFIRMATION=y +CONFIG_FEATURE_XARGS_SUPPORT_QUOTES=y +CONFIG_FEATURE_XARGS_SUPPORT_TERMOPT=y +CONFIG_FEATURE_XARGS_SUPPORT_ZERO_TERM=y + +# +# Init Utilities +# +CONFIG_INIT=y +# CONFIG_DEBUG_INIT is not set +CONFIG_FEATURE_USE_INITTAB=y +CONFIG_FEATURE_INIT_SCTTY=y +CONFIG_FEATURE_INIT_SYSLOG=y +CONFIG_FEATURE_EXTRA_QUIET=y +CONFIG_FEATURE_INIT_COREDUMPS=y +CONFIG_FEATURE_INITRD=y +CONFIG_HALT=y +CONFIG_MESG=y + +# +# Login/Password Management Utilities +# +CONFIG_FEATURE_SHADOWPASSWDS=y +CONFIG_USE_BB_SHADOW=y +CONFIG_USE_BB_PWD_GRP=y +CONFIG_ADDGROUP=y +CONFIG_FEATURE_ADDUSER_TO_GROUP=y +CONFIG_DELGROUP=y +CONFIG_FEATURE_DEL_USER_FROM_GROUP=y +CONFIG_ADDUSER=y +CONFIG_DELUSER=y +CONFIG_GETTY=y +CONFIG_FEATURE_UTMP=y +CONFIG_FEATURE_WTMP=y +CONFIG_LOGIN=y +CONFIG_PAM=y +CONFIG_LOGIN_SCRIPTS=y +CONFIG_FEATURE_NOLOGIN=y +CONFIG_FEATURE_SECURETTY=y +CONFIG_PASSWD=y +CONFIG_FEATURE_PASSWD_WEAK_CHECK=y +CONFIG_CRYPTPW=y +CONFIG_CHPASSWD=y +CONFIG_SU=y +CONFIG_FEATURE_SU_SYSLOG=y +CONFIG_FEATURE_SU_CHECKS_SHELLS=y +# CONFIG_SULOGIN is not set +CONFIG_VLOCK=y + +# +# Linux Ext2 FS Progs +# +CONFIG_CHATTR=y +CONFIG_FSCK=y +CONFIG_LSATTR=y + +# +# Linux Module Utilities +# +CONFIG_INSMOD=y +CONFIG_FEATURE_INSMOD_VERSION_CHECKING=y +CONFIG_FEATURE_INSMOD_KSYMOOPS_SYMBOLS=y +CONFIG_FEATURE_INSMOD_LOADINKMEM=y +CONFIG_FEATURE_INSMOD_LOAD_MAP=y +CONFIG_FEATURE_INSMOD_LOAD_MAP_FULL=y +CONFIG_RMMOD=y +CONFIG_LSMOD=y +CONFIG_FEATURE_LSMOD_PRETTY_2_6_OUTPUT=y +CONFIG_MODPROBE=y +CONFIG_FEATURE_MODPROBE_MULTIPLE_OPTIONS=y +CONFIG_FEATURE_MODPROBE_FANCY_ALIAS=y + +# +# Options common to multiple modutils +# +CONFIG_FEATURE_CHECK_TAINTED_MODULE=y +CONFIG_FEATURE_2_4_MODULES=y +CONFIG_FEATURE_2_6_MODULES=y +# CONFIG_FEATURE_QUERY_MODULE_INTERFACE is not set + +# +# Linux System Utilities +# +CONFIG_DMESG=y +CONFIG_FEATURE_DMESG_PRETTY=y +CONFIG_FBSET=y +CONFIG_FEATURE_FBSET_FANCY=y +CONFIG_FEATURE_FBSET_READMODE=y +CONFIG_FDFLUSH=y +CONFIG_FDFORMAT=y +CONFIG_FDISK=y +CONFIG_FDISK_SUPPORT_LARGE_DISKS=y +CONFIG_FEATURE_FDISK_WRITABLE=y +CONFIG_FEATURE_AIX_LABEL=y +CONFIG_FEATURE_SGI_LABEL=y +CONFIG_FEATURE_SUN_LABEL=y +CONFIG_FEATURE_OSF_LABEL=y +CONFIG_FEATURE_FDISK_ADVANCED=y +CONFIG_FREERAMDISK=y +CONFIG_FSCK_MINIX=y +CONFIG_MKFS_MINIX=y + +# +# Minix filesystem support +# +CONFIG_FEATURE_MINIX2=y +CONFIG_GETOPT=y +CONFIG_HEXDUMP=y +CONFIG_HWCLOCK=y +CONFIG_FEATURE_HWCLOCK_LONG_OPTIONS=y +CONFIG_FEATURE_HWCLOCK_ADJTIME_FHS=y +CONFIG_IPCRM=y +CONFIG_IPCS=y +CONFIG_LOSETUP=y +CONFIG_MDEV=y +CONFIG_FEATURE_MDEV_CONF=y +CONFIG_FEATURE_MDEV_EXEC=y +CONFIG_FEATURE_MDEV_LOAD_FIRMWARE=y +CONFIG_MKSWAP=y +CONFIG_FEATURE_MKSWAP_V0=y +CONFIG_MORE=y +CONFIG_FEATURE_USE_TERMIOS=y +CONFIG_MOUNT=y +CONFIG_FEATURE_MOUNT_HELPERS=y +CONFIG_FEATURE_MOUNT_NFS=y +CONFIG_FEATURE_MOUNT_CIFS=y +CONFIG_FEATURE_MOUNT_FLAGS=y +CONFIG_FEATURE_MOUNT_FSTAB=y +CONFIG_PIVOT_ROOT=y +CONFIG_RDATE=y +CONFIG_READPROFILE=y +CONFIG_SETARCH=y +CONFIG_SWAPONOFF=y +CONFIG_SWITCH_ROOT=y +CONFIG_UMOUNT=y +CONFIG_FEATURE_UMOUNT_ALL=y + +# +# Common options for mount/umount +# +CONFIG_FEATURE_MOUNT_LOOP=y +CONFIG_FEATURE_MTAB_SUPPORT=y + +# +# Miscellaneous Utilities +# +CONFIG_ADJTIMEX=y +CONFIG_BBCONFIG=y +CONFIG_CHRT=y +CONFIG_CROND=y +# CONFIG_DEBUG_CROND_OPTION is not set +CONFIG_FEATURE_CROND_CALL_SENDMAIL=y +# CONFIG_CRONTAB is not set +# CONFIG_DC is not set +CONFIG_DEVFSD=y +CONFIG_DEVFSD_MODLOAD=y +CONFIG_DEVFSD_FG_NP=y +CONFIG_DEVFSD_VERBOSE=y +CONFIG_FEATURE_DEVFS=y +CONFIG_EJECT=y +CONFIG_LAST=y +CONFIG_LESS=y +CONFIG_FEATURE_LESS_MAXLINES=9999999 +CONFIG_FEATURE_LESS_BRACKETS=y +CONFIG_FEATURE_LESS_FLAGS=y +CONFIG_FEATURE_LESS_FLAGCS=y +CONFIG_FEATURE_LESS_MARKS=y +CONFIG_FEATURE_LESS_REGEXP=y +CONFIG_HDPARM=y +CONFIG_FEATURE_HDPARM_GET_IDENTITY=y +CONFIG_FEATURE_HDPARM_HDIO_SCAN_HWIF=y +CONFIG_FEATURE_HDPARM_HDIO_UNREGISTER_HWIF=y +CONFIG_FEATURE_HDPARM_HDIO_DRIVE_RESET=y +CONFIG_FEATURE_HDPARM_HDIO_TRISTATE_HWIF=y +CONFIG_FEATURE_HDPARM_HDIO_GETSET_DMA=y +CONFIG_MAKEDEVS=y +# CONFIG_FEATURE_MAKEDEVS_LEAF is not set +CONFIG_FEATURE_MAKEDEVS_TABLE=y +CONFIG_MICROCOM=y +CONFIG_MOUNTPOINT=y +CONFIG_MT=y +CONFIG_RAIDAUTORUN=y +CONFIG_READAHEAD=y +CONFIG_RUNLEVEL=y +CONFIG_RX=y +CONFIG_STRINGS=y +CONFIG_SETSID=y +# CONFIG_TASKSET is not set +# CONFIG_FEATURE_TASKSET_FANCY is not set +CONFIG_TIME=y +CONFIG_TTYSIZE=y +CONFIG_WATCHDOG=y + +# +# Networking Utilities +# +CONFIG_FEATURE_IPV6=y +CONFIG_VERBOSE_RESOLUTION_ERRORS=y +CONFIG_ARP=y +CONFIG_ARPING=y +CONFIG_DNSD=y +CONFIG_ETHER_WAKE=y +CONFIG_FAKEIDENTD=y +# CONFIG_FTPGET is not set +# CONFIG_FTPPUT is not set +# CONFIG_FEATURE_FTPGETPUT_LONG_OPTIONS is not set +CONFIG_HOSTNAME=y +CONFIG_HTTPD=y +CONFIG_FEATURE_HTTPD_RANGES=y +CONFIG_FEATURE_HTTPD_USE_SENDFILE=y +CONFIG_FEATURE_HTTPD_RELOAD_CONFIG_SIGHUP=y +CONFIG_FEATURE_HTTPD_SETUID=y +CONFIG_FEATURE_HTTPD_BASIC_AUTH=y +CONFIG_FEATURE_HTTPD_AUTH_MD5=y +CONFIG_FEATURE_HTTPD_CONFIG_WITH_MIME_TYPES=y +CONFIG_FEATURE_HTTPD_CGI=y +CONFIG_FEATURE_HTTPD_CONFIG_WITH_SCRIPT_INTERPR=y +CONFIG_FEATURE_HTTPD_SET_REMOTE_PORT_TO_ENV=y +CONFIG_FEATURE_HTTPD_ENCODE_URL_STR=y +CONFIG_FEATURE_HTTPD_ERROR_PAGES=y +CONFIG_FEATURE_HTTPD_PROXY=y +CONFIG_IFCONFIG=y +CONFIG_FEATURE_IFCONFIG_STATUS=y +CONFIG_FEATURE_IFCONFIG_SLIP=y +CONFIG_FEATURE_IFCONFIG_MEMSTART_IOADDR_IRQ=y +CONFIG_FEATURE_IFCONFIG_HW=y +CONFIG_FEATURE_IFCONFIG_BROADCAST_PLUS=y +CONFIG_IFUPDOWN=y +CONFIG_IFUPDOWN_IFSTATE_PATH="/var/run/ifstate" +CONFIG_FEATURE_IFUPDOWN_IP=y +CONFIG_FEATURE_IFUPDOWN_IP_BUILTIN=y +# CONFIG_FEATURE_IFUPDOWN_IFCONFIG_BUILTIN is not set +CONFIG_FEATURE_IFUPDOWN_IPV4=y +CONFIG_FEATURE_IFUPDOWN_IPV6=y +CONFIG_FEATURE_IFUPDOWN_MAPPING=y +CONFIG_FEATURE_IFUPDOWN_EXTERNAL_DHCP=y +# CONFIG_INETD is not set +# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_ECHO is not set +# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_DISCARD is not set +# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_TIME is not set +# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_DAYTIME is not set +# CONFIG_FEATURE_INETD_SUPPORT_BUILTIN_CHARGEN is not set +# CONFIG_FEATURE_INETD_RPC is not set +CONFIG_IP=y +CONFIG_FEATURE_IP_ADDRESS=y +CONFIG_FEATURE_IP_LINK=y +CONFIG_FEATURE_IP_ROUTE=y +CONFIG_FEATURE_IP_TUNNEL=y +CONFIG_FEATURE_IP_RULE=y +CONFIG_FEATURE_IP_SHORT_FORMS=y +CONFIG_FEATURE_IP_RARE_PROTOCOLS=y +CONFIG_IPADDR=y +CONFIG_IPLINK=y +CONFIG_IPROUTE=y +CONFIG_IPTUNNEL=y +CONFIG_IPRULE=y +# CONFIG_IPCALC is not set +# CONFIG_FEATURE_IPCALC_FANCY is not set +# CONFIG_FEATURE_IPCALC_LONG_OPTIONS is not set +CONFIG_NAMEIF=y +CONFIG_NC=y +CONFIG_NC_SERVER=y +CONFIG_NC_EXTRA=y +CONFIG_NETSTAT=y +CONFIG_FEATURE_NETSTAT_WIDE=y +CONFIG_NSLOOKUP=y +CONFIG_PING=y +CONFIG_PING6=y +CONFIG_PSCAN=y +CONFIG_FEATURE_FANCY_PING=y +CONFIG_ROUTE=y +CONFIG_SLATTACH=y +CONFIG_TELNET=y +CONFIG_FEATURE_TELNET_TTYPE=y +CONFIG_FEATURE_TELNET_AUTOLOGIN=y +CONFIG_TELNETD=y +CONFIG_FEATURE_TELNETD_STANDALONE=y +# CONFIG_TFTP is not set +# CONFIG_FEATURE_TFTP_GET is not set +# CONFIG_FEATURE_TFTP_PUT is not set +# CONFIG_FEATURE_TFTP_BLOCKSIZE is not set +# CONFIG_DEBUG_TFTP is not set +CONFIG_TRACEROUTE=y +CONFIG_FEATURE_TRACEROUTE_VERBOSE=y +CONFIG_FEATURE_TRACEROUTE_SOURCE_ROUTE=y +CONFIG_FEATURE_TRACEROUTE_USE_ICMP=y +CONFIG_APP_UDHCPD=y +CONFIG_APP_DHCPRELAY=y +CONFIG_APP_DUMPLEASES=y +CONFIG_FEATURE_UDHCPD_WRITE_LEASES_EARLY=y +CONFIG_APP_UDHCPC=y +# CONFIG_FEATURE_UDHCP_DEBUG is not set +CONFIG_FEATURE_RFC3397=y +CONFIG_VCONFIG=y +CONFIG_WGET=y +CONFIG_FEATURE_WGET_STATUSBAR=y +CONFIG_FEATURE_WGET_AUTHENTICATION=y +CONFIG_FEATURE_WGET_LONG_OPTIONS=y +CONFIG_ZCIP=y + +# +# Process Utilities +# +CONFIG_FREE=y +CONFIG_FUSER=y +CONFIG_KILL=y +CONFIG_KILLALL=y +CONFIG_KILLALL5=y +CONFIG_NMETER=y +CONFIG_PGREP=y +CONFIG_PIDOF=y +CONFIG_FEATURE_PIDOF_SINGLE=y +CONFIG_FEATURE_PIDOF_OMIT=y +CONFIG_PKILL=y +CONFIG_PS=y +CONFIG_FEATURE_PS_WIDE=y +CONFIG_RENICE=y +CONFIG_BB_SYSCTL=y +CONFIG_TOP=y +CONFIG_FEATURE_TOP_CPU_USAGE_PERCENTAGE=y +CONFIG_FEATURE_TOP_CPU_GLOBAL_PERCENTS=y +CONFIG_FEATURE_TOP_DECIMALS=y +CONFIG_FEATURE_TOPMEM=y +CONFIG_UPTIME=y +CONFIG_WATCH=y + +# +# Shells +# +CONFIG_FEATURE_SH_IS_ASH=y +# CONFIG_FEATURE_SH_IS_HUSH is not set +# CONFIG_FEATURE_SH_IS_LASH is not set +# CONFIG_FEATURE_SH_IS_MSH is not set +# CONFIG_FEATURE_SH_IS_NONE is not set +CONFIG_ASH=y + +# +# Ash Shell Options +# +CONFIG_ASH_JOB_CONTROL=y +CONFIG_ASH_READ_NCHARS=y +CONFIG_ASH_READ_TIMEOUT=y +CONFIG_ASH_ALIAS=y +CONFIG_ASH_MATH_SUPPORT=y +CONFIG_ASH_MATH_SUPPORT_64=y +CONFIG_ASH_GETOPTS=y +CONFIG_ASH_BUILTIN_ECHO=y +CONFIG_ASH_BUILTIN_TEST=y +CONFIG_ASH_CMDCMD=y +CONFIG_ASH_MAIL=y +CONFIG_ASH_OPTIMIZE_FOR_SIZE=y +CONFIG_ASH_RANDOM_SUPPORT=y +CONFIG_ASH_EXPAND_PRMT=y +# CONFIG_HUSH is not set +# CONFIG_HUSH_HELP is not set +# CONFIG_HUSH_INTERACTIVE is not set +# CONFIG_HUSH_JOB is not set +# CONFIG_HUSH_TICK is not set +# CONFIG_HUSH_IF is not set +# CONFIG_HUSH_LOOPS is not set +# CONFIG_LASH is not set +# CONFIG_MSH is not set + +# +# Bourne Shell Options +# +CONFIG_FEATURE_SH_EXTRA_QUIET=y +CONFIG_FEATURE_SH_STANDALONE=y +CONFIG_CTTYHACK=y + +# +# System Logging Utilities +# +CONFIG_SYSLOGD=y +CONFIG_FEATURE_ROTATE_LOGFILE=y +CONFIG_FEATURE_REMOTE_LOG=y +CONFIG_FEATURE_IPC_SYSLOG=y +CONFIG_FEATURE_IPC_SYSLOG_BUFFER_SIZE=16 +CONFIG_LOGREAD=y +CONFIG_FEATURE_LOGREAD_REDUCED_LOCKING=y +CONFIG_KLOGD=y +CONFIG_LOGGER=y + +# +# Runit Utilities +# +CONFIG_RUNSV=y +CONFIG_RUNSVDIR=y +CONFIG_SV=y +CONFIG_SVLOGD=y +CONFIG_CHPST=y +CONFIG_SETUIDGID=y +CONFIG_ENVUIDGID=y +CONFIG_ENVDIR=y +CONFIG_SOFTLIMIT=y +# CONFIG_CHCON is not set +# CONFIG_FEATURE_CHCON_LONG_OPTIONS is not set +# CONFIG_GETENFORCE is not set +# CONFIG_GETSEBOOL is not set +# CONFIG_LOAD_POLICY is not set +# CONFIG_MATCHPATHCON is not set +# CONFIG_RESTORECON is not set +# CONFIG_RUNCON is not set +# CONFIG_FEATURE_RUNCON_LONG_OPTIONS is not set +# CONFIG_SELINUXENABLED is not set +# CONFIG_SETENFORCE is not set +# CONFIG_SETFILES is not set +# CONFIG_FEATURE_SETFILES_CHECK_OPTION is not set +# CONFIG_SETSEBOOL is not set + +# +# ipsvd utilities +# +CONFIG_TCPSVD=y +CONFIG_UDPSVD=y Index: branches/eam_branches/20090522/portage/suidctl.conf =================================================================== --- branches/eam_branches/20090522/portage/suidctl.conf (revision 24557) +++ branches/eam_branches/20090522/portage/suidctl.conf (revision 24557) @@ -0,0 +1,96 @@ +/bin/ping +/bin/su +/usr/bin/crontab +/usr/bin/dotlockfile +/usr/bin/fusermount +/usr/bin/man +/usr/bin/mutt_dotlock +/usr/bin/screen +/usr/bin/slocate +/usr/bin/sudo +/usr/bin/sudoedit +/usr/lib64/misc/utempter/utempter +/usr/libexec/lockspool +/usr/nagios/libexec/check_icmp +/usr/sbin/suexec2 +/usr/sbin/unix_chkpwd +## -rws--x--x 1 root root 1.1M Aug 9 14:33 /usr/bin/sperl5.8.8 +#/usr/bin/sperl5.8.8 +## -rws--x--x 1 root root 1.7M Aug 9 14:44 /usr/bin/Xorg +#/usr/bin/Xorg +## -r-s--x--x 1 root root 52K Feb 13 05:13 /sbin/mount.nfs +#/sbin/mount.nfs +## -rws--x--x 1 root root 39K Feb 13 05:05 /bin/umount +#/bin/umount +## -rws--x--x 1 root root 56K Feb 13 05:05 /bin/mount +#/bin/mount +## -rwxr-s--x 1 root tty 15K Feb 13 05:05 /usr/bin/write +#/usr/bin/write +## -rws--x--x 1 root root 31K Feb 12 19:27 /bin/ping6 +#/bin/ping6 +## -rws--x--x 1 root root 19K Feb 12 19:27 /usr/sbin/traceroute6 +#/usr/sbin/traceroute6 +## -rws--x--x 1 root root 23K Feb 13 01:00 /sbin/unix_chkpwd +#/sbin/unix_chkpwd +## -rws--x--x 1 root root 11K Feb 13 09:03 /usr/lib64/misc/glibc/pt_chown +#/usr/lib64/misc/glibc/pt_chown +## -rws--x--x 1 root root 9.4K Feb 13 09:03 /usr/lib32/misc/glibc/pt_chown +#/usr/lib32/misc/glibc/pt_chown +## -rws--x--x 1 root root 34K Feb 13 01:22 /bin/passwd +#/bin/passwd +## -rws--x--x 1 root root 28K Feb 13 01:22 /usr/bin/chsh +#/usr/bin/chsh +## -rws--x--x 1 root root 46K Feb 13 01:22 /usr/bin/chage +#/usr/bin/chage +## -rws--x--x 1 root root 38K Feb 13 01:22 /usr/bin/gpasswd +#/usr/bin/gpasswd +## -rws--x--x 1 root root 28K Feb 13 01:22 /usr/bin/newgrp +#/usr/bin/newgrp +## -rws--x--x 1 root root 37K Feb 13 01:22 /usr/bin/chfn +#/usr/bin/chfn +## -rws--x--x 1 root root 24K Feb 13 01:22 /usr/bin/expiry +#/usr/bin/expiry +## -rws--x--x 1 root root 19K Feb 13 02:25 /usr/bin/rcp +#/usr/bin/rcp +## -rws--x--x 1 root root 11K Feb 13 02:25 /usr/bin/rsh +#/usr/bin/rsh +## -rws--x--x 1 root root 15K Feb 13 02:25 /usr/bin/rlogin +#/usr/bin/rlogin +## -rws--x--x 1 root root 173K Feb 13 01:29 /usr/lib64/misc/ssh-keysign +#/usr/lib64/misc/ssh-keysign +## -rwxr-s--x 1 root mail 14K Feb 13 04:47 /usr/libexec/camel-lock-helper-1.2 +#/usr/libexec/camel-lock-helper-1.2 +## -rws--x--x 1 root root 14K Feb 13 01:33 /usr/bin/lppasswd +#/usr/bin/lppasswd +## -rws--x--x 1 root root 22K Feb 13 02:55 /usr/bin/smbmnt +#/usr/bin/smbmnt +## -rws--x--x 1 root root 22K Feb 13 02:55 /usr/bin/smbumount +#/usr/bin/smbumount +## -rwxr-s--x 1 root utmp 14K Feb 13 02:27 /usr/libexec/gnome-pty-helper +#/usr/libexec/gnome-pty-helper +## -r-xr-s--x 1 root games 113K Feb 13 11:27 /usr/bin/mahjongg +#/usr/bin/mahjongg +## -r-xr-s--x 1 root games 98K Feb 13 11:27 /usr/bin/gnotski +#/usr/bin/gnotski +## -r-xr-s--x 1 root games 95K Feb 13 11:27 /usr/bin/gnotravex +#/usr/bin/gnotravex +## -r-xr-s--x 1 root games 428K Feb 13 11:27 /usr/bin/gnibbles +#/usr/bin/gnibbles +## -r-xr-s--x 1 root games 105K Feb 13 11:27 /usr/bin/glines +#/usr/bin/glines +## -r-xr-s--x 1 root games 93K Feb 13 11:27 /usr/bin/same-gnome +#/usr/bin/same-gnome +## -r-xr-s--x 1 root games 107K Feb 13 11:27 /usr/bin/gtali +#/usr/bin/gtali +## -r-xr-s--x 1 root games 161K Feb 13 11:27 /usr/bin/gnobots2 +#/usr/bin/gnobots2 +## -r-xr-s--x 1 root games 125K Feb 13 11:27 /usr/bin/gnometris +#/usr/bin/gnometris +## -r-xr-s--x 1 root games 106K Feb 13 11:27 /usr/bin/gnomine +#/usr/bin/gnomine +## -rws--x--x 1 root root 18K Feb 13 07:19 /usr/bin/cpufreq-selector +#/usr/bin/cpufreq-selector +## -rws--x--- 1 root apache 14K Apr 1 12:06 /usr/sbin/suexec +#/usr/sbin/suexec +## -rws--x--x 1 games root 19K Feb 13 02:31 /usr/libexec/emacs/22.1/x86_64-pc-linux-gnu/update-game-score +#/usr/libexec/emacs/22.1/x86_64-pc-linux-gnu/update-game-score Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Abstract-Meta-Class/Abstract-Meta-Class-0.11.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Abstract-Meta-Class/Abstract-Meta-Class-0.11.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Abstract-Meta-Class/Abstract-Meta-Class-0.11.ebuild (revision 24557) @@ -0,0 +1,22 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Abstract-Meta-Class-0.11 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Abstract-Meta-Class&mode=dist" +SRC_URI="mirror://cpan/authors/id/A/AD/ADRIANWIT/Abstract-Meta-Class-0.11.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/Test-Pod-Coverage + dev-perl/Test-Pod + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Abstract-Meta-Class/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Abstract-Meta-Class/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Abstract-Meta-Class/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Abstract-Meta-Class-0.11.tar.gz 16789 RMD160 94e00dfa7328152c0ec57fe23b2a58312aa947de SHA1 931ded91dd7a390d38a5c3a5ec236f7004035778 SHA256 7b87bc53175daeeefc1474cd6aadde1c4ba3f85db23ffd9358766d68fe1ee6b4 +EBUILD Abstract-Meta-Class-0.11.ebuild 547 RMD160 4c6e3916cff27b8e97d92f85b76509b41445d59a SHA1 4d03f846245e0681341802512a28dd4b00308c03 SHA256 1bdb97a48f20f27ccb1e6a3cadfe2a9f5bb319bb4776242099b0030c60d6a435 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Astro-FITS-CFITSIO/Astro-FITS-CFITSIO-1.05.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Astro-FITS-CFITSIO/Astro-FITS-CFITSIO-1.05.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Astro-FITS-CFITSIO/Astro-FITS-CFITSIO-1.05.ebuild (revision 24557) @@ -0,0 +1,19 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Astro-FITS-CFITSIO-1.05 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Astro-FITS-CFITSIO&mode=dist" +SRC_URI="mirror://cpan/authors/id/P/PR/PRATZLAFF/Astro-FITS-CFITSIO-1.05.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Astro-FITS-CFITSIO/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Astro-FITS-CFITSIO/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Astro-FITS-CFITSIO/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Astro-FITS-CFITSIO-1.05.tar.gz 194185 RMD160 6e3efb00a00ca52438eb336b5c5b8f79d28a6a40 SHA1 46ba3a40078dcbc559b64552a3f29418f0581e2e SHA256 bbeb6fd3b17b290842a88a8179c2526abae21fa67172295385a742b51bbfab70 +EBUILD Astro-FITS-CFITSIO-1.05.ebuild 474 RMD160 aca6ec00252e4e66f4c1d8e911162fbf97b49d02 SHA1 0eab94b83dd548b408e912c9f8a2bac08a33e86e SHA256 24b201c20a481b94daf1e4aea717283b6fb10cbed65db6b6363be654bfb1e339 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-Scoped/Config-Scoped-0.12.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-Scoped/Config-Scoped-0.12.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-Scoped/Config-Scoped-0.12.ebuild (revision 24557) @@ -0,0 +1,22 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Config-Scoped-0.12 + +DESCRIPTION="feature rich configuration file parser" +HOMEPAGE="http://search.cpan.org/search?query=Config-Scoped&mode=dist" +SRC_URI="mirror://cpan/authors/id/G/GA/GAISSMAI/Config-Scoped-0.12.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/Error + >=dev-perl/Parse-RecDescent-1.94 + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-Scoped/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-Scoped/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-Scoped/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Config-Scoped-0.12.tar.gz 67631 RMD160 1ad59a6849d2b0574646977b9fcfa088841e289d SHA1 5a1040e703784ef9ea18468f8fedceae4ae6f85b SHA256 c5213a51508fe345f37b0a288836f2697a5928f3b119420653388a597016c512 +EBUILD Config-Scoped-0.12.ebuild 545 RMD160 a01d2f72d0c59d1f5c0199d93f7d564a84b83c86 SHA1 9ddea0771de9dc47449dba7648dec94fefc8f582 SHA256 7c29d0fd47dc1cf5cd8e0aa884e485fd1fc8df716884b2a2dfeaca1f530a3f78 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-YAML/Config-YAML-1.42.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-YAML/Config-YAML-1.42.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-YAML/Config-YAML-1.42.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Config-YAML-1.42 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Config-YAML&mode=dist" +SRC_URI="mirror://cpan/authors/id/M/MD/MDXI/Config-YAML-1.42.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND=">=dev-perl/yaml-0.68 + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-YAML/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-YAML/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Config-YAML/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Config-YAML-1.42.tar.gz 5963 RMD160 b9f1c8bbabcb91829f56febc08ed53cfff9c2ffc SHA1 86c430b32275095bce870e08c98b7ddd8660fb15 SHA256 447cc03f25b81d5d3b63076ffa45b837cfebb504b1f390dff8d3104091a42f62 +EBUILD Config-YAML-1.42.ebuild 493 RMD160 f2483290de0967c84e81e03b429d08b62f3883ee SHA1 b956b5c9a3d3d480091a5d9e331ecc9446e15043 SHA256 07839e8b51e65ec2be5c2d3c1ec0068811938408710ccf7d2305bf51098afe7b Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DB-File/DB-File-1.820.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DB-File/DB-File-1.820.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DB-File/DB-File-1.820.ebuild (revision 24557) @@ -0,0 +1,19 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/DB_File-1.820 + +DESCRIPTION="Tie to DB files" +HOMEPAGE="http://search.cpan.org/search?query=DB-File&mode=dist" +SRC_URI="mirror://cpan/authors/id/P/PM/PMQS/DB_File-1.820.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DB-File/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DB-File/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DB-File/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST DB_File-1.820.tar.gz 88298 RMD160 0e20492868bad21b38c717420a4e9642ffedd3a6 SHA1 f0a5311913cd059095977eee3441c947f1632748 SHA256 eae8d2d2144504118773f3e1787321d2c757e7c5abf0a60591c73495352ddf4a +EBUILD DB-File-1.820.ebuild 429 RMD160 4d997cfc2b001fb3b6eec6980b0bbec1f0c05959 SHA1 81453efde4162254c3b6b8c5f7cee82885d32fca SHA256 ec7c823f47cd9e6d719a898a2c8ecb70d2f2488b43c3256a81d152100eebf2b5 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DBIx-Connection/DBIx-Connection-0.08.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DBIx-Connection/DBIx-Connection-0.08.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DBIx-Connection/DBIx-Connection-0.08.ebuild (revision 24557) @@ -0,0 +1,24 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/DBIx-Connection-0.08 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=DBIx-Connection&mode=dist" +SRC_URI="mirror://cpan/authors/id/A/AD/ADRIANWIT/DBIx-Connection-0.08.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/Test-Pod-Coverage + perl-gcpan/Abstract-Meta-Class + dev-perl/DBI + dev-perl/Test-Pod + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DBIx-Connection/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DBIx-Connection/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DBIx-Connection/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST DBIx-Connection-0.08.tar.gz 19923 RMD160 d91f177d0ba5612510ac87d18a123b135302a5a6 SHA1 73eeb8eb5e2eaa3cae54b2c62870853e8b114fbb SHA256 e73cc185431a8a02ba33c71f4791fa63add1b1a001fe0b3c7d8bcf6ece86829c +EBUILD DBIx-Connection-0.08.ebuild 581 RMD160 6a820960c4f82fe37a5161e8b8ee196d088b9561 SHA1 4700b24a982fc0a782fd6a0fa2ac1f65e27d5c18 SHA256 729b35dec442d91a81570df41ea909beff99b94fcacd7f07027a2cb014c0e84e Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-Domain/Data-Validate-Domain-0.09.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-Domain/Data-Validate-Domain-0.09.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-Domain/Data-Validate-Domain-0.09.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Data-Validate-Domain-0.09 + +DESCRIPTION="Light weight module for validating domains" +HOMEPAGE="http://search.cpan.org/search?query=Data-Validate-Domain&mode=dist" +SRC_URI="mirror://cpan/authors/id/N/NE/NEELY/Data-Validate-Domain-0.09.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/Net-Domain-TLD + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-Domain/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-Domain/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-Domain/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Data-Validate-Domain-0.09.tar.gz 7848 RMD160 9d7bf643911d060a9e4e59ddff9d4e47f24e5cea SHA1 034162d818affdaebbbb1daff9fe6424db2fdd8c SHA256 57a2511cd937cc2699c7f723bd34d9cacf3db6351b4d667279c63938d300bd82 +EBUILD Data-Validate-Domain-0.09.ebuild 542 RMD160 f4381cb081f24e85266d3c502068d7b22e0ceba6 SHA1 4c2ec75e42726a186f7ddaf6038c783f104d3b06 SHA256 1122bf06c35bea761e101c80266070066d36797719ed194dfca434f8a19f861b Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Data-Validate-IP-0.08.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Data-Validate-IP-0.08.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Data-Validate-IP-0.08.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Data-Validate-IP-0.08 + +DESCRIPTION="Lightweight IP validation module." +HOMEPAGE="http://search.cpan.org/search?query=Data-Validate-IP&mode=dist" +SRC_URI="mirror://cpan/authors/id/N/NE/NEELY/Data-Validate-IP-0.08.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/Net-Netmask + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Data-Validate-IP-0.10.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Data-Validate-IP-0.10.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Data-Validate-IP-0.10.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Data-Validate-IP-0.10 + +DESCRIPTION="Lightweight IP validation module." +HOMEPAGE="http://search.cpan.org/search?query=Data-Validate-IP&mode=dist" +SRC_URI="mirror://cpan/authors/id/N/NE/NEELY/Data-Validate-IP-0.10.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/Net-Netmask + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-IP/Manifest (revision 24557) @@ -0,0 +1,4 @@ +DIST Data-Validate-IP-0.08.tar.gz 6299 RMD160 e557d2432123ad8646e70f97d9960dea5a671e25 SHA1 774132b82525f059d5e902baf77dad5d0d659f85 SHA256 c2dafe09ab292b95a9eb2863507eafd7aa52b054d4fbdafd713bcb9bb320b83d +DIST Data-Validate-IP-0.10.tar.gz 7446 RMD160 8df8a8b80c1499ff9dd6a97fe676ddb07a6c6f27 SHA1 9fdb953131e9f480ddd839a48924501fbd9614e8 SHA256 0b9dec2e63f54ec957a7b242b818f6d04cc1da0f7f27541a87990fdfaab81215 +EBUILD Data-Validate-IP-0.08.ebuild 518 RMD160 09d96b78a489ffe91fe66a17ac12c72a58b3d936 SHA1 ea23295452b8b4f394fd9fd8a2183878dccb46be SHA256 05d4d2325ee36ef322a8cc3fe9189059a03c4590ca103cc31fa198b0c850fcb8 +EBUILD Data-Validate-IP-0.10.ebuild 518 RMD160 0f8848b7b72f04865d7d062b6129b31ff81a2b44 SHA1 0ca09dda4193c28589f23952e1ed735bdf23cf70 SHA256 dad2a7fa11c1917f486640e3c40c3e6929b933b77ed37b34aedab270ed973a1c Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-URI/Data-Validate-URI-0.05.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-URI/Data-Validate-URI-0.05.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-URI/Data-Validate-URI-0.05.ebuild (revision 24557) @@ -0,0 +1,22 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Data-Validate-URI-0.05 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Data-Validate-URI&mode=dist" +SRC_URI="mirror://cpan/authors/id/S/SO/SONNEN/Data-Validate-URI-0.05.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="perl-gcpan/Data-Validate-Domain + perl-gcpan/Data-Validate-IP + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-URI/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-URI/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Data-Validate-URI/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Data-Validate-URI-0.05.tar.gz 10409 RMD160 f00ade1cba42ead1afd36dd8c49ba02ecc68a194 SHA1 8d14517338bf68b64282e07f6b7abb605cad9f97 SHA256 1c7c61a5cf9a1e3fd50da3a61d0fcf1f2419343c908b71d9ee56bc7ba24755d0 +EBUILD Data-Validate-URI-0.05.ebuild 553 RMD160 d0b5b787f8a43be9440d70bbb012e7aeab971211 SHA1 657542e9a29e1bb4a403a71dae5fdee4c4b277e5 SHA256 52645d5fe8a1b2d8e0a9077f8b372c802f85402f7ba6df5d0908a70f98073545 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/DateTime-Format-Strptime-1.0702.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/DateTime-Format-Strptime-1.0702.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/DateTime-Format-Strptime-1.0702.ebuild (revision 24557) @@ -0,0 +1,24 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/DateTime-Format-Strptime-1.0702 + +DESCRIPTION="Parse and Format DateTimes using Strptime" +HOMEPAGE="http://search.cpan.org/search?query=DateTime-Format-Strptime&mode=dist" +SRC_URI="mirror://cpan/authors/id/R/RI/RICKM/DateTime-Format-Strptime-1.0702.tgz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/DateTime + dev-perl/DateTime-Locale + dev-perl/DateTime-TimeZone + dev-perl/Params-Validate + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/DateTime-Format-Strptime-1.0901.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/DateTime-Format-Strptime-1.0901.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/DateTime-Format-Strptime-1.0901.ebuild (revision 24557) @@ -0,0 +1,24 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/DateTime-Format-Strptime-1.0901 + +DESCRIPTION="Parse and Format DateTimes using Strptime" +HOMEPAGE="http://search.cpan.org/search?query=DateTime-Format-Strptime&mode=dist" +SRC_URI="mirror://cpan/authors/id/R/RI/RICKM/DateTime-Format-Strptime-1.0901.tgz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND=">=perl-gcpan/DateTime-0.4302 + >=dev-perl/DateTime-Locale-0.42 + dev-perl/DateTime-TimeZone + >=dev-perl/Params-Validate-0.91 + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime-Format-Strptime/Manifest (revision 24557) @@ -0,0 +1,4 @@ +DIST DateTime-Format-Strptime-1.0702.tgz 26655 RMD160 5d4bc9d3e3cf0aace81598e87af0620b40237613 SHA1 cc104e9c124712aefed3e7349823b0b373c3a709 SHA256 60cf7edf4e76006f1b85f4c8c9558194d1d8ae1eff111a67f64b9920185783e8 +DIST DateTime-Format-Strptime-1.0901.tgz 28481 RMD160 b6735e479ee91f5712e1d10f1d3b39ee265d5c03 SHA1 6ca67d2a536a435f1e1b521d42a5310a1737860c SHA256 de78a12ceaa3c5795322f3f8b2bfc726a5b914029c7ac9615bc3c6e021285d40 +EBUILD DateTime-Format-Strptime-1.0702.ebuild 628 RMD160 bd09a6d22dd87f224cf1558cb720b909c8860102 SHA1 adc3200075175d811ac092a8b5aebdc0a9494d6b SHA256 46b34d528234ec604014309469cfc29b35ff11fb6b412a04a28f7b36b847cb22 +EBUILD DateTime-Format-Strptime-1.0901.ebuild 653 RMD160 d0f5a866f475ef1be892d24217b0b8a83d31e429 SHA1 5b6a31c2ec6c76dc1954f3fe4555cbc91d8ef75e SHA256 9dec9013fcd379fb55b6da899f6f8d025e1c9246250429b60d198f3c12486d42 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/DateTime-0.4302.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/DateTime-0.4302.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/DateTime-0.4302.ebuild (revision 24557) @@ -0,0 +1,23 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/DateTime-0.4302 + +DESCRIPTION="A complete, easy to use date and time object" +HOMEPAGE="http://search.cpan.org/search?query=DateTime&mode=dist" +SRC_URI="mirror://cpan/authors/id/D/DR/DROLSKY/DateTime-0.4302.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/DateTime-Locale + dev-perl/DateTime-TimeZone + dev-perl/Params-Validate + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/DateTime-0.50.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/DateTime-0.50.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/DateTime-0.50.ebuild (revision 24557) @@ -0,0 +1,23 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/DateTime-0.50 + +DESCRIPTION="A complete, easy to use date and time object" +HOMEPAGE="http://search.cpan.org/search?query=DateTime&mode=dist" +SRC_URI="mirror://cpan/authors/id/D/DR/DROLSKY/DateTime-0.50.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND=">=dev-perl/DateTime-Locale-0.42 + dev-perl/DateTime-TimeZone + >=dev-perl/Params-Validate-0.91 + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/DateTime/Manifest (revision 24557) @@ -0,0 +1,4 @@ +DIST DateTime-0.4302.tar.gz 100707 RMD160 3031f46f3e8ee437995f7c71e1627c667d5d60ab SHA1 2c62f53373c87f5221cda0dbe62705ef90b615ed SHA256 050aa0f97cc14a93558e5dbc7d4a3e160e6083aff39fb6b770617a60533346d9 +DIST DateTime-0.50.tar.gz 137754 RMD160 8ce6016216cc6b2ae8e1b18d95c9986c43af426b SHA1 20eea7116ad8a7a1995f6ff5e4984372f5d79a01 SHA256 5b96a033589be67f58f8fcd037f651ac2b2a3da832720fd8c79c38e79c31c752 +EBUILD DateTime-0.4302.ebuild 569 RMD160 fe0eb57587b9fdf76a7577deb27ac068428ea051 SHA1 add9cb86e21be28049311721be529a3f69f27a20 SHA256 43f477a79d39f2f94a21ba2c7bdf2cff7793b6975a59a37163fd038c20b00a39 +EBUILD DateTime-0.50.ebuild 579 RMD160 80fb73bac8b166c4387514daba46c572cacac075 SHA1 4bf386fbeb0979f87823bdb518b021f760b588ad SHA256 baf111bf076b0805352911a69fbb99e4344277ebe2cfb5390f5b93dc9f7df933 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Digest-MD5-File-0.06.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Digest-MD5-File-0.06.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Digest-MD5-File-0.06.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Digest-MD5-File-0.06 + +DESCRIPTION="Digest::MD5 Sub Class to use paths directly" +HOMEPAGE="http://search.cpan.org/search?query=Digest-MD5-File&mode=dist" +SRC_URI="mirror://cpan/authors/id/D/DM/DMUEY/Digest-MD5-File-0.06.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/libwww-perl + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Digest-MD5-File-0.07.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Digest-MD5-File-0.07.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Digest-MD5-File-0.07.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Digest-MD5-File-0.07 + +DESCRIPTION="Digest::MD5 Sub Class to use paths directly" +HOMEPAGE="http://search.cpan.org/search?query=Digest-MD5-File&mode=dist" +SRC_URI="mirror://cpan/authors/id/D/DM/DMUEY/Digest-MD5-File-0.07.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/libwww-perl + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Digest-MD5-File/Manifest (revision 24557) @@ -0,0 +1,4 @@ +DIST Digest-MD5-File-0.06.tar.gz 4948 RMD160 5f3ac547dcc9db65d665542ee472a2a430991a02 SHA1 a62e151d7ce7feff845b1b5c2b14db0ec5d4f436 SHA256 54791e960e589f3a367e3b5d317f3e0b2fabb544307f432e467813db4ef544a4 +DIST Digest-MD5-File-0.07.tar.gz 5122 RMD160 8f009fd02769b190e49e6e44ae37b600f84832f5 SHA1 d24593883e5df3ddc5cf2645fa40f1f62942286b SHA256 1282d15451f3eab38603305bc0519777891b63d9d577c8532e07112c815cc1a6 +EBUILD Digest-MD5-File-0.06.ebuild 525 RMD160 7cffa3570d8890c5f9020d8dcfc6abd971061850 SHA1 431fa00f90b3a947f63218e73b8ebd669944af16 SHA256 29a8b261fc7f25375c395be4256d07adc86c2275f70bc76ea912ca972820a35e +EBUILD Digest-MD5-File-0.07.ebuild 525 RMD160 693a7913a7ece88124f33cb6eab4a2f57e89c756 SHA1 ea3a850985fdfd205df5b819b02c3756493f0140 SHA256 ab038d12890dd4d5f765945b61792bc22ce3efb95abb4fd3c88082c1fe64cee1 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-ExtAttr/File-ExtAttr-1.09.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-ExtAttr/File-ExtAttr-1.09.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-ExtAttr/File-ExtAttr-1.09.ebuild (revision 24557) @@ -0,0 +1,20 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/File-ExtAttr-1.09 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=File-ExtAttr&mode=dist" +SRC_URI="mirror://cpan/authors/id/R/RI/RICHDAWE/File-ExtAttr-1.09.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-ExtAttr/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-ExtAttr/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-ExtAttr/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST File-ExtAttr-1.09.tar.gz 36426 RMD160 64b210fd591240e7eac3457a70bc66d4bb5aed01 SHA1 490c6d6ff5a621cc05279ec0a72bc3396204a7ff SHA256 e39a3f9784da4844b22d6a44739d9f96178d4fc0fbaa6bfe0a420f883c90103c +EBUILD File-ExtAttr-1.09.ebuild 478 RMD160 f7b9f4d1f216f2218b1a2cca0ed0deef3ef29626 SHA1 012db0357fa845e883baeb81b4c68dd7e2225784 SHA256 c3696495fa662330c46fccab02836d8f34fb1b468bfb35b5902634ab5feed827 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-Mountpoint/File-Mountpoint-0.01.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-Mountpoint/File-Mountpoint-0.01.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-Mountpoint/File-Mountpoint-0.01.ebuild (revision 24557) @@ -0,0 +1,20 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/File-Mountpoint-0.01 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=File-Mountpoint&mode=dist" +SRC_URI="mirror://cpan/authors/id/J/JH/JHOBLITT/File-Mountpoint-0.01.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-Mountpoint/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-Mountpoint/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/File-Mountpoint/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST File-Mountpoint-0.01.tar.gz 4063 RMD160 940f16b18a80ea0ffa5fe784de6fd50aa4da8c85 SHA1 3677b7979372804641041a52d488c46a6343f22f SHA256 888c74afe558d031127561ebe8f639a5cf9d5f3244261c30cd510c8df314faa8 +EBUILD File-Mountpoint-0.01.ebuild 487 RMD160 76748c195fd725545b8f47e05fb020bffd48579c SHA1 e022d132eaad1fcd2cc32cebeccc35060162ee66 SHA256 b8b04a461757ae36d331e2ec8844b77414b0ceeb2e82f1d784186d84e3703ce7 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/IO-Compress/IO-Compress-2.020.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/IO-Compress/IO-Compress-2.020.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/IO-Compress/IO-Compress-2.020.ebuild (revision 24557) @@ -0,0 +1,22 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/IO-Compress-2.020 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=IO-Compress&mode=dist" +SRC_URI="mirror://cpan/authors/id/P/PM/PMQS/IO-Compress-2.020.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="virtual/perl-Compress-Raw-Bzip2 + virtual/perl-Compress-Raw-Zlib + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/IO-Compress/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/IO-Compress/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/IO-Compress/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST IO-Compress-2.020.tar.gz 204495 RMD160 9e9b85a9701eb6abaf6ee18a63e46db25abe08cf SHA1 cd7871852f6c19ed4be61822f620fb706f01498f SHA256 dad5874a1f6bde93467b570e2821aaf530c073219ab138efb8ffb3ed67075bcb +EBUILD IO-Compress-2.020.ebuild 538 RMD160 2e7e1bcf417a86fa8900bd3f8e2512acbe014826 SHA1 95ea7356ecf0c35517bf4b6cc5c2b6243e6ba683 SHA256 3025bac6502b756c720c4c3cc54fb5d785ff7ac2f5741662f88de7ee689508aa Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-CoreList/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-CoreList/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-CoreList/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Module-CoreList-2.17.tar.gz 42602 RMD160 b4f4f6e950ab2790a9fe378a4b56c49751e7cf60 SHA1 8390235120aa4258834524569f9deddca674d972 SHA256 aafcbe4aa74870bf51575dd31bfd6f277602d306b132445ea01344281f8cae3c +EBUILD Module-CoreList-2.17.ebuild 486 RMD160 c60608bd5d521a62056d46afc651b9969d9d3de0 SHA1 619e3754b9ec4d74c3af34fc16cb269a1ab95fcc SHA256 189f5c20ec6a79214925bc64c30b7d4baa74a691f6fa71229e22d4ed33ab7e0b Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-CoreList/Module-CoreList-2.17.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-CoreList/Module-CoreList-2.17.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-CoreList/Module-CoreList-2.17.ebuild (revision 24557) @@ -0,0 +1,20 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Module-CoreList-2.17 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Module-CoreList&mode=dist" +SRC_URI="mirror://cpan/authors/id/R/RG/RGARCIA/Module-CoreList-2.17.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Manifest (revision 24557) @@ -0,0 +1,4 @@ +DIST Module-Load-0.12.tar.gz 4429 RMD160 71a81898b3db3b7463ef79c497dac1c8e749aec8 SHA1 3592fef6d127bac74146dc5d7572b3fd31d1ddeb SHA256 e36705699c3c9a3b815c71fb4176481e4d3c070eeb9f8144e2dae5ba2e7c6d1e +DIST Module-Load-0.16.tar.gz 5095 RMD160 676def147471d6116c770aa31e812e45236c98d7 SHA1 142197478036a7db50897a906a331380a34cbda9 SHA256 f05c3300c988dc5f5f02d55f8ee241c05fbf4565c512f804f47c6e72e5f69e33 +EBUILD Module-Load-0.12.ebuild 471 RMD160 95c4f3eeb11f08650fa4fe2746a2007391533235 SHA1 43b62701730c5af44cf739643116a806af1049fa SHA256 fec3c668dfe3607e983e0bfdb0f57cb3882cf5f80c73e6f8092207ccccd33fbb +EBUILD Module-Load-0.16.ebuild 471 RMD160 4228b7f6d2377914ae36566e270572cc75e905cc SHA1 ebf2ad869f8d3e34a15143500fe9f3f9eda19346 SHA256 16f73b05e7229856d692bb900e26e2a9e38af78bc4667c2de5a883927b600e4e Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Module-Load-0.12.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Module-Load-0.12.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Module-Load-0.12.ebuild (revision 24557) @@ -0,0 +1,20 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Module-Load-0.12 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Module-Load&mode=dist" +SRC_URI="mirror://cpan/authors/id/K/KA/KANE/Module-Load-0.12.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Module-Load-0.16.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Module-Load-0.16.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Module-Load/Module-Load-0.16.ebuild (revision 24557) @@ -0,0 +1,20 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Module-Load-0.16 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Module-Load&mode=dist" +SRC_URI="mirror://cpan/authors/id/K/KA/KANE/Module-Load-0.16.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Net-HTTPServer/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Net-HTTPServer/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Net-HTTPServer/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Net-HTTPServer-1.1.1.tar.gz 65401 RMD160 b1e2fc6bb65c98db931df72edeb91919d28f00db SHA1 2bd379fae579d5b1e16de374b06d44cacdd900d7 SHA256 980bf68ad3ceb5fac3ae47447a06a402d54f018b426e54c49803c39cbb419a94 +EBUILD Net-HTTPServer-1.1.1.ebuild 499 RMD160 4d4e64fc0af9ca7a745dbcefd866f24e1e4a6842 SHA1 b945968349edab938aead112cf17353fa5eb31c8 SHA256 38de930c4afd5b7ebfe6d32ff38572772c367705695a1d5a31586af2bca1c038 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Net-HTTPServer/Net-HTTPServer-1.1.1.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Net-HTTPServer/Net-HTTPServer-1.1.1.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Net-HTTPServer/Net-HTTPServer-1.1.1.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Net-HTTPServer-1.1.1 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Net-HTTPServer&mode=dist" +SRC_URI="mirror://cpan/authors/id/R/RE/REATMON/Net-HTTPServer-1.1.1.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/URI + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/SQL-Interp/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/SQL-Interp/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/SQL-Interp/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST SQL-Interp-1.06.tar.gz 36211 RMD160 3e8dd30d538bccb6f521fcbf4f87dbba55eeb9a7 SHA1 8e3aa0b95eb11a0065c24f846c167716d2e9ceaa SHA256 130ef97e309f00d75a057ecb0c4c3ca56ab996b1d763e79921ee7c401bb44cf1 +EBUILD SQL-Interp-1.06.ebuild 486 RMD160 b0617fa0cb8bae4185932f150997b7947c5d8fdd SHA1 9132a038a76c21f741987f98d389e1143f8d9db5 SHA256 91ad0957660b1a2a3a9714feef2f944cebcd511f9a139cf271c9f297b99389af Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/SQL-Interp/SQL-Interp-1.06.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/SQL-Interp/SQL-Interp-1.06.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/SQL-Interp/SQL-Interp-1.06.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/SQL-Interp-1.06 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=SQL-Interp&mode=dist" +SRC_URI="mirror://cpan/authors/id/M/MA/MARKSTOS/SQL-Interp-1.06.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/DBI + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Simple-SAX-Serializer/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Simple-SAX-Serializer/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Simple-SAX-Serializer/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Simple-SAX-Serializer-0.05.tar.gz 8520 RMD160 5666b1e753d91203b19cdc2d3d971ad7e0639d12 SHA1 7d3bd618989c021d504f03e5b6e10cf530347a9e SHA256 28d6e9f078616701a6c040c3cd4e073f1a5dbd03dcc8cca58ef6cf1e5a86808a +EBUILD Simple-SAX-Serializer-0.05.ebuild 603 RMD160 58442a2fa9ce50890d5dafbe263264c19e8c5141 SHA1 8ab1cb6d4a223e4c09d8642b53b569c85bb8c176 SHA256 3ddcffba0f2215661c0d4d90058dde3ba083c80e77df3ac00d36c6cd644f3678 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Simple-SAX-Serializer/Simple-SAX-Serializer-0.05.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Simple-SAX-Serializer/Simple-SAX-Serializer-0.05.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Simple-SAX-Serializer/Simple-SAX-Serializer-0.05.ebuild (revision 24557) @@ -0,0 +1,24 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Simple-SAX-Serializer-0.05 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Simple-SAX-Serializer&mode=dist" +SRC_URI="mirror://cpan/authors/id/A/AD/ADRIANWIT/Simple-SAX-Serializer-0.05.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/XML-SAX + dev-perl/Test-Pod-Coverage + perl-gcpan/Abstract-Meta-Class + dev-perl/Test-Pod + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Cmd/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Cmd/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Cmd/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Test-Cmd-1.05.tar.gz 31104 RMD160 a8dbc9ed20cff3b1e16e93e898bad90d99c18dea SHA1 32bb30c7120f74b3a9fed4ce3109fed31babb55d SHA256 335449a95d81cc16e4dc67e7bcfc02745faf4d02f7f431e0b19c469f331fd3be +EBUILD Test-Cmd-1.05.ebuild 458 RMD160 98d2ed756107866304f88c5daee65e05674d82f9 SHA1 e86eebbd590097a050bd7841b34fa54808ca8624 SHA256 aee0f8e574488198d6165fb5e0f2626981b5647f3ab393d7c3f14fbc24302d65 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Cmd/Test-Cmd-1.05.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Cmd/Test-Cmd-1.05.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Cmd/Test-Cmd-1.05.ebuild (revision 24557) @@ -0,0 +1,19 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Test-Cmd-1.05 + +DESCRIPTION="Portable test infrastructure for commands" +HOMEPAGE="http://search.cpan.org/search?query=Test-Cmd&mode=dist" +SRC_URI="mirror://cpan/authors/id/K/KN/KNIGHT/Test-Cmd-1.05.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-DBUnit/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-DBUnit/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-DBUnit/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Test-DBUnit-0.20.tar.gz 33087 RMD160 727a1f027ba80be3f1933ac5f3840617cbfe94be SHA1 01c383b9ca4160222108285322dccd79453dc65d SHA256 55208acc7dfe7899672a6fde4d6d0f639cf5d2a5173b6dd7b6460dbfcc761660 +EBUILD Test-DBUnit-0.20.ebuild 692 RMD160 225804672ffceb0f6f7563c356017ec73a04a119 SHA1 4350267f8abf18a7897729448a26239a854a12c4 SHA256 b283bab31c4d29f8e65df8d5141dc19db01317b90c955af64603c2eafae5bf48 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-DBUnit/Test-DBUnit-0.20.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-DBUnit/Test-DBUnit-0.20.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-DBUnit/Test-DBUnit-0.20.ebuild (revision 24557) @@ -0,0 +1,28 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Test-DBUnit-0.20 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Test-DBUnit&mode=dist" +SRC_URI="mirror://cpan/authors/id/A/AD/ADRIANWIT/Test-DBUnit-0.20.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/XML-SAX + dev-perl/Test-Pod-Coverage + dev-perl/Test-Pod + dev-perl/XML-Writer + >=perl-gcpan/DBIx-Connection-0.08 + >=perl-gcpan/Simple-SAX-Serializer-0.05 + dev-perl/Sub-Uplevel + perl-gcpan/Abstract-Meta-Class + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Distribution/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Distribution/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Distribution/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Test-Distribution-2.00.tar.gz 11326 RMD160 8bed19b2363364230c61f76213223e702d39c23a SHA1 547c00dde9cd68656d4d53ca285bb6a5e69c9ba6 SHA256 50ecd07fb6cab7b2b2a73b5526ee43b90e77734c9bcec95ce8822b9c0a912b68 +EBUILD Test-Distribution-2.00.ebuild 635 RMD160 776e45be35c0efa04c19b6b8b97277d34b369f51 SHA1 e5d370534b43e9e0dce17aac0afddcfaf446e8f5 SHA256 4c9d8d9d42d6ff7df84488cb9e360ebad15c70a2ffd76b9f4ed3877d89c219ef Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Distribution/Test-Distribution-2.00.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Distribution/Test-Distribution-2.00.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-Distribution/Test-Distribution-2.00.ebuild (revision 24557) @@ -0,0 +1,25 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Test-Distribution-2.00 + +DESCRIPTION="No description available" +HOMEPAGE="http://search.cpan.org/search?query=Test-Distribution&mode=dist" +SRC_URI="mirror://cpan/authors/id/S/SR/SRSHAH/Test-Distribution-2.00.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND=">=dev-perl/Pod-Coverage-0.19 + >=dev-perl/File-Find-Rule-0.30 + dev-perl/Test-Pod-Coverage + >=perl-gcpan/Module-CoreList-2.17 + dev-perl/Test-Pod + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-URI/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-URI/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-URI/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Test-URI-1.08.tar.gz 5142 RMD160 98d1bd8607fa776bc01d74aabccef89eba5c5c24 SHA1 ee08f478321d9ba4768ff18db2a327f6afcb17d0 SHA256 0688d30da58cc4c4f63887c5474d279b39295afdea8ed836136e1a6818b11162 +EBUILD Test-URI-1.08.ebuild 490 RMD160 f7448ad4e161afcfad82f28c4a43a902dec78a05 SHA1 aedcd78be1d0598f36535245e5f45088e0bef575 SHA256 6d9c7a68262439ed7d0f07d0a806f7ab36bb25255f68f490a772e2400381485a Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-URI/Test-URI-1.08.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-URI/Test-URI-1.08.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Test-URI/Test-URI-1.08.ebuild (revision 24557) @@ -0,0 +1,21 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Test-URI-1.08 + +DESCRIPTION="check parts of URI with Test::Builder" +HOMEPAGE="http://search.cpan.org/search?query=Test-URI&mode=dist" +SRC_URI="mirror://cpan/authors/id/B/BD/BDFOY/Test-URI-1.08.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-perl/URI + dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Tree-DAG-Node/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Tree-DAG-Node/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Tree-DAG-Node/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST Tree-DAG_Node-1.06.tar.gz 32843 RMD160 b3011e10cbfe8b1dbc7d3ee191e6dbe692b18d46 SHA1 8f412087b33942df06f20ca7c2e8ea0bd2354742 SHA256 502570774dd3bf11345fb028ca4cdd79f04db12ee68a826f0c44e24da0e3db2a +EBUILD Tree-DAG-Node-1.06.ebuild 475 RMD160 bf895dcbd28e90178591b9598c224dd2aae4310d SHA1 dc09e3b9ce78e069f5250cdeaffc818994f195d5 SHA256 4386835ce06a1c5044dbf6fa9697ebf8d3de49742530f624d53cd97f29707964 Index: branches/eam_branches/20090522/portage_overlay/perl-gcpan/Tree-DAG-Node/Tree-DAG-Node-1.06.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/perl-gcpan/Tree-DAG-Node/Tree-DAG-Node-1.06.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/perl-gcpan/Tree-DAG-Node/Tree-DAG-Node-1.06.ebuild (revision 24557) @@ -0,0 +1,20 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# This ebuild generated by g-cpan 0.15.0 + +inherit perl-module + +S=${WORKDIR}/Tree-DAG_Node-1.06 + +DESCRIPTION="base class for trees" +HOMEPAGE="http://search.cpan.org/search?query=Tree-DAG-Node&mode=dist" +SRC_URI="mirror://cpan/authors/id/C/CO/COGENT/Tree-DAG_Node-1.06.tar.gz" + + +IUSE="" + +SLOT="0" +LICENSE="|| ( Artistic GPL-2 )" +KEYWORDS="amd64" + +DEPEND="dev-lang/perl" Index: branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/Manifest (revision 24557) @@ -0,0 +1,2 @@ +DIST fv44_src.tar.gz 33380333 RMD160 ad75a579d9b7812fd6b4d783165668993a7715fa SHA1 df9fe6bbb790ac95c653609192b1e675ee90697d SHA256 862ec9c878ef17fee5a2955c78a4250a23d0f31539f3439713f2975356f9a978 +EBUILD fv-4.4.ebuild 693 RMD160 a66a924bfdfa11aec016d284fb098ec54427d6b8 SHA1 cedbe681f67a5461c16490a00280ac205645bd35 SHA256 8f1d7eb34d79230c2029fb5db5bc3f926eabb5990a80e30ae7f1ad1bdea968d3 Index: branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/files/digest-fv-4.4 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/files/digest-fv-4.4 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/files/digest-fv-4.4 (revision 24557) @@ -0,0 +1,3 @@ +MD5 fbc784053fc8ff3f57f01b464129ed6f fv44_src.tar.gz 33380333 +RMD160 ad75a579d9b7812fd6b4d783165668993a7715fa fv44_src.tar.gz 33380333 +SHA256 862ec9c878ef17fee5a2955c78a4250a23d0f31539f3439713f2975356f9a978 fv44_src.tar.gz 33380333 Index: branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/fv-4.4.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/fv-4.4.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sci-astronomy/fv/fv-4.4.ebuild (revision 24557) @@ -0,0 +1,25 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sci-astronomy/fv/fv-4.4.ebuild,v 1.1 2007/08/08 03:23:33 jhoblitt Exp $ + +DESCRIPTION="programs for manipulating and analyzing FITS data file" +HOMEPAGE="http://heasarc.gsfc.nasa.gov/ftools/fv/fv_download.html" +SRC_URI="http://heasarc.gsfc.nasa.gov/FTP/software/lheasoft/fv/fv44_src.tar.gz" + +LICENSE="" +SLOT="0" +KEYWORDS="x86 amd64" +IUSE="" + +DEPEND="dev-tcltk/itcl" +RDEPEND="" + +S="${WORKDIR}/headas-6.1/BUILD_DIR/" + +src_compile() { + econf \ + --prefix=/opt/headas-6.1 \ + || die "econf failed" + + emake || die "emake failed" +} Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/Manifest (revision 24557) @@ -0,0 +1,5 @@ +AUX archttp 518 RMD160 0f0e7044be8904bed4f492dcbd3257c6920f5f74 SHA1 80937c13b679806fedb1bbfd2d59ec4c33f8ef3b SHA256 3e8cd943ff80794e543759b4a3f76061f3d9d310e98226cd483dd004473c8ebf +DIST archttp64 1750040 RMD160 2f7139b3586ab5c7b46b96c6f98db49826c423ee SHA1 0cba54ddab839645357238f54218b42e7c209077 SHA256 d35339123236439d8bf0ddebc878b33f85ee76da1b02cd1d74845a4bd3181799 +EBUILD archttp-1.81.250.ebuild 940 RMD160 57dc7990cbe81ac092339d4b46e411d8487cce4a SHA1 7c2dfe417fccde32a7e0fe7ae01f19f508a335a4 SHA256 cb458a41239ea9a3f4d1a935f35d909430c6ddba4f81fcd7eb32fa0ece7e9293 +EBUILD archttp-1.82.70911.ebuild 831 RMD160 44a46347e9c9d5838cbf0055f1c6c155ca7dbccb SHA1 2d3116b18a490b1aa6720bace7c97800f8f34d71 SHA256 2ff70bbdd4babecba9cac25350068d820e0f96984a9eb8372624a68c36880da9 +MISC archttpsrv.conf 80 RMD160 e9b3342404b6e39dac9bbd7a1927bc2457004b21 SHA1 d1e42bb82794af77703d872354c8b3bd7be4587b SHA256 94483001552a1f502b163b82e21e5ad62d497ac40708c17194ba3cec458b3bc7 Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttp-1.81.250.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttp-1.81.250.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttp-1.81.250.ebuild (revision 24557) @@ -0,0 +1,32 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/project/hw/portage-overlay/sys-apps/archttp/archttp-1.81.250.ebuild,v 1.2 2009/06/23 02:16:13 jhoblitt Exp $ + +inherit eutils toolchain-funcs flag-o-matic + +DESCRIPTION="Areca management HTTP proxy" +HOMEPAGE="http://freshmeat.net/projects/sysvinit/" +#SRC_URI="ftp://ftp.areca.com.tw/RaidCards/AP_Drivers/Linux/HTTP/V1.81.250_70306/x86_64/archttp64" +SRC_URI="ftp://ftp.areca.com.tw/RaidCards/AP_Drivers/Linux/HTTP/OldVersions/V1.81.250_70306/x86_64/archttp64" + +LICENSE="public-domain" +RESTRICT="nomirror" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +RDEPEND="" +DEPEND="" + +src_unpack() { + true +} + +src_install() { + mkdir -p ${D}/usr/sbin + mkdir -p ${D}/etc/init.d + cp ${DISTDIR}/archttp64 ${D}/usr/sbin + cp ${FILESDIR}/archttp ${D}/etc/init.d + chmod 0755 ${D}/usr/sbin/archttp64 + chmod 0755 ${D}/etc/init.d/archttp +} Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttp-1.82.70911.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttp-1.82.70911.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttp-1.82.70911.ebuild (revision 24557) @@ -0,0 +1,31 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/archttp/archttp-1.82.70911.ebuild,v 1.1 2008/06/18 21:29:03 jhoblitt Exp $ + +inherit eutils toolchain-funcs flag-o-matic + +DESCRIPTION="Areca management HTTP proxy" +HOMEPAGE="http://freshmeat.net/projects/sysvinit/" +SRC_URI="ftp://ftp.areca.com.tw/RaidCards/AP_Drivers/Linux/HTTP/V1.82_70911/x86-64/archttp64" + +LICENSE="public-domain" +RESTRICT="nomirror" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +RDEPEND="" +DEPEND="" + +src_unpack() { + true +} + +src_install() { + mkdir -p ${D}/usr/sbin + mkdir -p ${D}/etc/init.d + cp ${DISTDIR}/archttp64 ${D}/usr/sbin + cp ${FILESDIR}/archttp ${D}/etc/init.d + chmod 0755 ${D}/usr/sbin/archttp64 + chmod 0755 ${D}/etc/init.d/archttp +} Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttpsrv.conf =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttpsrv.conf (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/archttpsrv.conf (revision 24557) @@ -0,0 +1,7 @@ +[GENERAL] +BindingIp=0.0.0.0 +HTTPPort=81 +ScanPci=YES +ScanRs232=NO +ScanInband=NO + Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/archttp =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/archttp (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/archttp (revision 24557) @@ -0,0 +1,21 @@ +#!/sbin/runscript + +depend() { + need net + use logger +} + +start() { +# archttp looks for and write in config file under /. as I can tell there is +# no way to specify the path to the config file. + ebegin "Starting Areca Webserver" + start-stop-daemon --start --quiet --exec /usr/sbin/archttp64 > /dev/null & + eend $? +} + +stop() { + ebegin "Stopping Areca Webserver" + start-stop-daemon --stop -s INT --retry timeout --exec /usr/sbin/archttp64 + sleep 5 + eend $? +} Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/digest-archttp-1.81.250 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/digest-archttp-1.81.250 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/digest-archttp-1.81.250 (revision 24557) @@ -0,0 +1,3 @@ +MD5 7ce56bcbebc5180a174012054b9a6c84 archttp64 1750040 +RMD160 2f7139b3586ab5c7b46b96c6f98db49826c423ee archttp64 1750040 +SHA256 d35339123236439d8bf0ddebc878b33f85ee76da1b02cd1d74845a4bd3181799 archttp64 1750040 Index: branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/digest-archttp-1.82.70911 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/digest-archttp-1.82.70911 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/archttp/files/digest-archttp-1.82.70911 (revision 24557) @@ -0,0 +1,3 @@ +MD5 7ce56bcbebc5180a174012054b9a6c84 archttp64 1750040 +RMD160 2f7139b3586ab5c7b46b96c6f98db49826c423ee archttp64 1750040 +SHA256 d35339123236439d8bf0ddebc878b33f85ee76da1b02cd1d74845a4bd3181799 archttp64 1750040 Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/Manifest (revision 24557) @@ -0,0 +1,5 @@ +EBUILD ipp-base-0.1.ebuild 3502 RMD160 c48f0aeb842a6c97f35812cc37b38a47cf37c203 SHA1 1ee3ba4f45c0ed1cab20e34bb7318cab1fcfdf2b SHA256 fddc7a9309a0283ab1dbc23d25ccf3cd8dfc5a0b8be82aea3ed78e27b7f8c332 +EBUILD ipp-base-0.2.ebuild 3540 RMD160 4ca4a64aa428040b6aa5e0f86cbba0341dfa2e98 SHA1 5c6d04920f52ac581d6f574ee7250c00c463fd46 SHA256 99d75b5b452f7377b04a779eb3c0442a87ad2df5fb0a783c0162f98308c60532 +EBUILD ipp-base-0.3.ebuild 4777 RMD160 ad1502ad1dbc611d5c7cd0d0f8c27942d7bfc0b1 SHA1 cf3d34684ee4bc1ca71467c8a2d4a2bfb4a5e00f SHA256 a2954602440c11ad2dbe8015e79576a25540ff979fff0161e150a41cc3342b33 +EBUILD ipp-base-0.4.ebuild 4837 RMD160 8346935b48c7add26258f0a0b0e3209e994a99df SHA1 eb57a9751aa779408b58895c1ad15d471944e87b SHA256 22d7f50fba5b395c18cb626616354c7047a62ef1beaca213cba5f4f9448d6245 +EBUILD ipp-base-0.5.ebuild 6932 RMD160 6f86e5c68a040c94d668b8ea0e0d18a22ba58d8e SHA1 4b3db22176eb0f2c61daf4105fffeeb979d59bf0 SHA256 0da0440fe7fd16624aa69c0c97c977ffb7a2e3b08537fd83ec1f805e668df7cb Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.1.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.1.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.1.ebuild (revision 24557) @@ -0,0 +1,169 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/ipp-base/ipp-base-0.1.ebuild,v 1.10 2007/11/30 03:11:01 jhoblitt Exp $ + +DESCRIPTION="base set of packages for IPP compute nodes" +HOMEPAGE="pan-starrs.ifa.hawaii.edu" +SRC_URI="" + +LICENSE="" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +DEPEND="" +RDEPEND=" + app-admin/denyhosts + app-admin/mcelog + app-admin/showconsole + app-admin/sudo + app-admin/syslog-ng + app-admin/sysstat + app-admin/tmpwatch + app-arch/pax + app-arch/rar + app-arch/sharutils + app-backup/rsnapshot + app-cdr/cdrtools + app-cdr/dvd+rw-tools + app-dicts/aspell-en + app-doc/doxygen + app-editors/emacs + app-editors/gvim + app-editors/vim + app-editors/xemacs + app-emulation/emul-linux-x86-baselibs + app-emulation/emul-linux-x86-compat + app-emulation/emul-linux-x86-gtklibs + app-emulation/emul-linux-x86-qtlibs + app-emulation/emul-linux-x86-sdl + app-misc/screen + app-office/openoffice-bin + app-portage/g-cpan + app-portage/gentoolkit + app-portage/layman + app-shells/bash-completion + app-shells/tcsh + app-text/aspell + app-text/tetex + app-text/tree + app-vim/gentoo-syntax + dev-db/mysql + dev-java/blackdown-jdk + dev-lang/mono + dev-lang/swig + dev-lang/tcl + dev-lang/tk + dev-perl/perltidy + dev-util/astyle + dev-util/catalyst + dev-util/ccache + dev-util/cogito + dev-util/cvs + dev-util/cvs2cl + dev-util/cvsps + dev-util/ddd + dev-util/diffstat + dev-util/efence + dev-util/git + dev-util/ltrace + dev-util/ltrace + dev-util/mono-tools + dev-util/oprofile + dev-util/strace + dev-util/subversion + dev-util/svk + dev-util/valgrind + gnome-base/gnome + mail-client/mailx + mail-client/mutt + media-gfx/gimp + media-gfx/gqview + media-gfx/povray + media-gfx/xfig + media-gfx/xv + media-libs/imlib + media-libs/jasper + media-libs/netpbm + net-analyzer/arping + net-analyzer/gnu-netcat + net-analyzer/iftop + net-analyzer/nagios-nrpe + net-analyzer/nagios-nsca + net-analyzer/nagios-plugins + net-analyzer/net-snmp + net-analyzer/nmap + net-analyzer/tcpdump + net-analyzer/traceroute + net-dialup/minicom + net-dns/bind-tools + net-fs/autofs + net-fs/nfs-utils + net-ftp/ftp + net-ftp/ncftp + net-misc/bridge-utils + net-misc/curl + net-misc/dhcpcd + net-misc/howl + net-misc/ifenslave + net-misc/iperf + net-misc/iputils + net-misc/netkit-rsh + net-misc/ntp + net-misc/rdate + net-misc/rdate + net-misc/rdesktop + net-misc/telnet-bsd + net-misc/vconfig + net-misc/vnc + net-misc/whois + net-nds/yp-tools + net-nds/ypbind + net-nds/ypserv + net-print/cups + net-www/apache + sci-libs/cfitsio + sci-libs/fftw + sci-libs/gsl + sci-visualization/gnuplot + sys-apps/dog + sys-apps/eject + sys-apps/ethtool + sys-apps/hotplug + sys-apps/ipmitool + sys-apps/iproute2 + sys-apps/lm_sensors + sys-apps/parted + sys-apps/sdparm + sys-apps/slocate + sys-apps/smartmontools + sys-apps/watchdog + sys-apps/xinetd + sys-boot/grub + sys-cluster/pvm + sys-devel/distcc + sys-devel/gdb + sys-devel/prelink + sys-fs/dosfstools + sys-fs/fuse + sys-fs/inotify-tools + sys-fs/jfsutils + sys-fs/lvm2 + sys-fs/sshfs-fuse + sys-fs/xfsprogs + sys-kernel/genkernel + sys-libs/libtermcap-compat + sys-power/acpitool + sys-power/apcupsd + sys-process/lsof + sys-process/schedtool + sys-process/schedutils + sys-process/vixie-cron + www-client/links + www-client/lynx + x11-base/xorg-x11 + x11-libs/xforms + x11-misc/x11vnc + x11-misc/xsensors + app-text/aspell +" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.2.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.2.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.2.ebuild (revision 24557) @@ -0,0 +1,172 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/ipp-base/ipp-base-0.2.ebuild,v 1.4 2007/11/30 03:11:01 jhoblitt Exp $ + +DESCRIPTION="base set of packages for IPP compute nodes" +HOMEPAGE="pan-starrs.ifa.hawaii.edu" +SRC_URI="" + +LICENSE="" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +DEPEND="" +RDEPEND=" + app-admin/denyhosts + app-admin/mcelog + app-admin/showconsole + app-admin/sudo + app-admin/syslog-ng + app-admin/sysstat + app-admin/tmpwatch + app-arch/pax + app-arch/rar + app-arch/rpm + app-arch/sharutils + app-backup/rsnapshot + app-cdr/cdrtools + app-cdr/dvd+rw-tools + app-dicts/aspell-en + app-doc/doxygen + app-editors/emacs + app-editors/gvim + app-editors/vim + app-editors/xemacs + app-emulation/emul-linux-x86-baselibs + app-emulation/emul-linux-x86-compat + app-emulation/emul-linux-x86-gtklibs + app-emulation/emul-linux-x86-qtlibs + app-emulation/emul-linux-x86-sdl + app-misc/screen + app-office/openoffice-bin + app-portage/g-cpan + app-portage/gentoolkit + app-portage/layman + app-shells/bash-completion + app-shells/tcsh + app-text/aspell + app-text/aspell + app-text/tetex + app-text/tree + app-vim/gentoo-syntax + dev-db/mysql + dev-java/blackdown-jdk + dev-lang/mono + dev-lang/php + dev-lang/swig + dev-lang/tcl + dev-lang/tk + dev-perl/perltidy + dev-util/astyle + dev-util/catalyst + dev-util/ccache + dev-util/cogito + dev-util/cvs + dev-util/cvs2cl + dev-util/cvsps + dev-util/ddd + dev-util/diffstat + dev-util/efence + dev-util/git + dev-util/ltrace + dev-util/ltrace + dev-util/mono-tools + dev-util/oprofile + dev-util/strace + dev-util/subversion + dev-util/svk + dev-util/valgrind + gnome-base/gnome + mail-client/mailx + mail-client/mutt + media-gfx/gimp + media-gfx/gqview + media-gfx/povray + media-gfx/xfig + media-gfx/xv + media-libs/imlib + media-libs/jasper + media-libs/netpbm + net-analyzer/arping + net-analyzer/gnu-netcat + net-analyzer/iftop + net-analyzer/nagios-nrpe + net-analyzer/nagios-nsca + net-analyzer/nagios-plugins + net-analyzer/net-snmp + net-analyzer/nmap + net-analyzer/tcpdump + net-analyzer/traceroute + net-dialup/minicom + net-dns/bind-tools + net-fs/autofs + net-fs/nfs-utils + net-ftp/ftp + net-ftp/ncftp + net-misc/bridge-utils + net-misc/curl + net-misc/dhcpcd + net-misc/howl + net-misc/ifenslave + net-misc/iperf + net-misc/iputils + net-misc/netkit-rsh + net-misc/ntp + net-misc/rdate + net-misc/rdate + net-misc/rdesktop + net-misc/telnet-bsd + net-misc/vconfig + net-misc/vnc + net-misc/whois + net-nds/yp-tools + net-nds/ypbind + net-nds/ypserv + net-print/cups + net-www/apache + sci-libs/cfitsio + sci-libs/fftw + sci-libs/gsl + sci-visualization/gnuplot + sys-apps/dog + sys-apps/eject + sys-apps/ethtool + sys-apps/hotplug + sys-apps/ipmitool + sys-apps/iproute2 + sys-apps/lm_sensors + sys-apps/parted + sys-apps/sdparm + sys-apps/slocate + sys-apps/smartmontools + sys-apps/watchdog + sys-apps/xinetd + sys-boot/grub + sys-cluster/pvm + sys-devel/bc + sys-devel/distcc + sys-devel/gdb + sys-devel/prelink + sys-fs/dosfstools + sys-fs/fuse + sys-fs/inotify-tools + sys-fs/jfsutils + sys-fs/lvm2 + sys-fs/sshfs-fuse + sys-fs/xfsprogs + sys-kernel/genkernel + sys-libs/libtermcap-compat + sys-power/acpitool + sys-power/apcupsd + sys-process/lsof + sys-process/schedtool + sys-process/schedutils + sys-process/vixie-cron + www-client/links + www-client/lynx + x11-base/xorg-x11 + x11-libs/xforms + x11-misc/x11vnc + x11-misc/xsensors +" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.3.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.3.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.3.ebuild (revision 24557) @@ -0,0 +1,221 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/ipp-base/ipp-base-0.3.ebuild,v 1.10 2008/06/20 21:15:19 jhoblitt Exp $ + +DESCRIPTION="base set of packages for IPP compute nodes" +HOMEPAGE="pan-starrs.ifa.hawaii.edu" +SRC_URI="" + +LICENSE="" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +DEPEND="" +RDEPEND=" + app-admin/denyhosts + app-admin/mcelog + app-admin/showconsole + app-admin/sudo + app-admin/syslog-ng + app-admin/sysstat + app-admin/tmpwatch + app-arch/pax + app-arch/rar + app-arch/rpm + app-arch/sharutils + app-backup/rsnapshot + app-cdr/dvd+rw-tools + app-dicts/aspell-en + app-doc/doxygen + app-editors/emacs + app-editors/gvim + app-editors/vim + app-editors/xemacs + app-emulation/emul-linux-x86-baselibs + app-emulation/emul-linux-x86-compat + app-emulation/emul-linux-x86-gtklibs + app-emulation/emul-linux-x86-qtlibs + app-emulation/emul-linux-x86-sdl + app-misc/screen + app-office/openoffice-bin + app-portage/g-cpan + app-portage/gentoolkit + app-portage/layman + app-shells/bash-completion + app-shells/sash + app-shells/tcsh + app-text/aspell + app-text/aspell + app-text/tetex + app-text/tree + app-vim/gentoo-syntax + dev-db/mysql + dev-java/blackdown-jdk + dev-lang/mono + dev-lang/php + dev-lang/swig + dev-lang/tcl + dev-lang/tk + dev-perl/Apache-Test + dev-perl/Array-Compare + dev-perl/Cache + dev-perl/Class-Accessor + dev-perl/Class-Factory-Util + dev-perl/DBD-mysql + dev-perl/DBI + dev-perl/DateManip + dev-perl/DateTime-Locale + dev-perl/DateTime-TimeZone + dev-perl/File-Find-Rule + dev-perl/File-NFSLock + dev-perl/File-Slurp + dev-perl/Heap + dev-perl/IPC-Run + dev-perl/JSON + dev-perl/Log-Log4perl + dev-perl/Net-Domain-TLD + dev-perl/Number-Compare + dev-perl/Params-Validate + dev-perl/Parse-RecDescent + dev-perl/Proc-Daemon + dev-perl/SOAP-Lite + dev-perl/Statistics-Descriptive + dev-perl/Template-Toolkit + dev-perl/Test-Warn + dev-perl/perltidy + dev-perl/version + dev-php/PEAR-DB + dev-python/numpy + dev-util/astyle + dev-util/catalyst + dev-util/ccache + dev-util/cvs + dev-util/cvs2cl + dev-util/cvsps + dev-util/ddd + dev-util/diffstat + dev-util/efence + dev-util/git + dev-util/ltrace + dev-util/ltrace + dev-util/mono-tools + dev-util/oprofile + dev-util/strace + dev-util/subversion + dev-util/svk + dev-util/valgrind + gnome-base/gnome + mail-client/mailx + mail-client/mutt + media-gfx/gimp + media-gfx/gqview + media-gfx/povray + media-gfx/xfig + media-gfx/xv + media-libs/imlib + media-libs/jasper + media-libs/netpbm + net-analyzer/arping + net-analyzer/gnu-netcat + net-analyzer/iftop + net-analyzer/nagios + net-analyzer/nagios-core + net-analyzer/nagios-imagepack + net-analyzer/nagios-nrpe + net-analyzer/nagios-nsca + net-analyzer/nagios-plugins + net-analyzer/net-snmp + net-analyzer/nmap + net-analyzer/tcpdump + net-analyzer/traceroute + net-dialup/minicom + net-dns/bind-tools + net-fs/autofs + net-fs/nfs-utils + net-ftp/ftp + net-ftp/ncftp + net-misc/bridge-utils + net-misc/curl + net-misc/dhcpcd + net-misc/ifenslave + net-misc/iperf + net-misc/iputils + net-misc/netkit-rsh + net-misc/ntp + net-misc/rdate + net-misc/rdate + net-misc/rdesktop + net-misc/telnet-bsd + net-misc/vconfig + net-misc/vnc + net-misc/whois + net-nds/yp-tools + net-nds/ypbind + net-nds/ypserv + net-print/cups + perl-gcpan/Astro-FITS-CFITSIO + perl-gcpan/Config-Scoped + perl-gcpan/Data-Validate-Domain + perl-gcpan/Data-Validate-URI + perl-gcpan/DateTime + perl-gcpan/DateTime-Format-Builder + perl-gcpan/DateTime-Format-ISO8601 + perl-gcpan/DateTime-Format-Strptime + perl-gcpan/Digest-MD5-File + perl-gcpan/IPC-Cmd + perl-gcpan/Module-Load + perl-gcpan/Module-Load-Conditional + perl-gcpan/Net-HTTPServer + perl-gcpan/Params-Check + perl-gcpan/Test-Cmd + sci-libs/cfitsio + sci-libs/fftw + sci-libs/gsl + sci-libs/pgplot + sci-visualization/gnuplot + sys-apps/dog + sys-apps/eject + sys-apps/ethtool + sys-apps/hotplug + sys-apps/ipmitool + sys-apps/iproute2 + sys-apps/irqbalance + sys-apps/lm_sensors + sys-apps/parted + sys-apps/sdparm + sys-apps/slocate + sys-apps/smartmontools + sys-apps/watchdog + sys-apps/xinetd + sys-boot/grub + sys-cluster/ganglia + sys-cluster/pvm + sys-devel/bc + sys-devel/distcc + sys-devel/gdb + sys-devel/prelink + sys-fs/dosfstools + sys-fs/fuse + sys-fs/inotify-tools + sys-fs/jfsutils + sys-fs/lvm2 + sys-fs/mdadm + sys-fs/sshfs-fuse + sys-fs/xfsprogs + sys-kernel/genkernel + sys-libs/libtermcap-compat + sys-power/acpitool + sys-power/apcupsd + sys-process/lsof + sys-process/schedtool + sys-process/vixie-cron + www-client/links + www-client/lynx + www-servers/apache + x11-base/xorg-x11 + x11-libs/xforms + x11-misc/x11vnc + x11-misc/xsensors + x11-terms/pssh +" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.4.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.4.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.4.ebuild (revision 24557) @@ -0,0 +1,224 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/ipp-base/ipp-base-0.4.ebuild,v 1.3 2008/09/24 20:54:30 jhoblitt Exp $ + +DESCRIPTION="base set of packages for IPP compute nodes" +HOMEPAGE="pan-starrs.ifa.hawaii.edu" +SRC_URI="" + +LICENSE="" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +DEPEND="" +RDEPEND=" + app-admin/denyhosts + app-admin/mcelog + app-admin/showconsole + app-admin/sudo + app-admin/syslog-ng + app-admin/sysstat + app-admin/tmpwatch + app-arch/pax + app-arch/rar + app-arch/rpm + app-arch/sharutils + app-backup/rsnapshot + app-cdr/dvd+rw-tools + app-dicts/aspell-en + app-doc/doxygen + app-editors/emacs + app-editors/gvim + app-editors/vim + app-editors/xemacs + app-emulation/emul-linux-x86-baselibs + app-emulation/emul-linux-x86-compat + app-emulation/emul-linux-x86-gtklibs + app-emulation/emul-linux-x86-qtlibs + app-emulation/emul-linux-x86-sdl + app-misc/screen + app-office/openoffice-bin + app-portage/g-cpan + app-portage/gentoolkit + app-portage/layman + app-shells/bash-completion + app-shells/sash + app-shells/tcsh + app-text/aspell + app-text/aspell + app-text/tetex + app-text/tree + app-vim/gentoo-syntax + dev-db/mysql + dev-java/blackdown-jdk + dev-lang/mono + dev-lang/php + dev-lang/swig + dev-lang/tcl + dev-lang/tk + dev-perl/Apache-Test + dev-perl/Array-Compare + dev-perl/Cache + dev-perl/Class-Accessor + dev-perl/Class-Factory-Util + dev-perl/DBD-mysql + dev-perl/DBI + dev-perl/DateManip + dev-perl/DateTime-Locale + dev-perl/DateTime-TimeZone + dev-perl/Email-Send + dev-perl/File-Find-Rule + dev-perl/File-NFSLock + dev-perl/File-Slurp + dev-perl/Heap + dev-perl/IPC-Run + dev-perl/JSON + dev-perl/Log-Log4perl + dev-perl/Net-Domain-TLD + dev-perl/Number-Compare + dev-perl/Params-Validate + dev-perl/Parse-RecDescent + dev-perl/Proc-Daemon + dev-perl/SOAP-Lite + dev-perl/Statistics-Descriptive + dev-perl/Template-Toolkit + dev-perl/Test-Warn + dev-perl/perltidy + dev-perl/version + dev-php/PEAR-DB + dev-python/numpy + dev-tcltk/expect + dev-util/astyle + dev-util/catalyst + dev-util/ccache + dev-util/cvs + dev-util/cvs2cl + dev-util/cvsps + dev-util/ddd + dev-util/diffstat + dev-util/efence + dev-util/git + dev-util/ltrace + dev-util/ltrace + dev-util/mono-tools + dev-util/oprofile + dev-util/strace + dev-util/subversion + dev-util/svk + dev-util/valgrind + gnome-base/gnome + mail-client/mailx + mail-client/mutt + media-gfx/gimp + media-gfx/gqview + media-gfx/povray + media-gfx/xfig + media-gfx/xv + media-libs/imlib + media-libs/jasper + media-libs/netpbm + net-analyzer/arping + net-analyzer/gnu-netcat + net-analyzer/iftop + net-analyzer/nagios + net-analyzer/nagios-core + net-analyzer/nagios-imagepack + net-analyzer/nagios-nrpe + net-analyzer/nagios-nsca + net-analyzer/nagios-plugins + net-analyzer/net-snmp + net-analyzer/nmap + net-analyzer/tcpdump + net-analyzer/traceroute + net-dialup/minicom + net-dns/bind-tools + net-fs/autofs + net-fs/nfs-utils + net-ftp/ftp + net-ftp/ncftp + net-misc/bridge-utils + net-misc/curl + net-misc/dhcpcd + net-misc/ifenslave + net-misc/iperf + net-misc/iputils + net-misc/netkit-rsh + net-misc/ntp + net-misc/rdate + net-misc/rdate + net-misc/rdesktop + net-misc/telnet-bsd + net-misc/vconfig + net-misc/vnc + net-misc/whois + net-nds/yp-tools + net-nds/ypbind + net-nds/ypserv + net-print/cups + perl-gcpan/Astro-FITS-CFITSIO + perl-gcpan/Config-Scoped + perl-gcpan/Data-Validate-Domain + perl-gcpan/Data-Validate-URI + perl-gcpan/DateTime + perl-gcpan/DateTime-Format-Builder + perl-gcpan/DateTime-Format-ISO8601 + perl-gcpan/DateTime-Format-Strptime + perl-gcpan/Digest-MD5-File + perl-gcpan/IPC-Cmd + perl-gcpan/Module-Load + perl-gcpan/Module-Load-Conditional + perl-gcpan/Net-HTTPServer + perl-gcpan/Params-Check + perl-gcpan/Test-Cmd + sci-libs/cfitsio + sci-libs/fftw + sci-libs/gsl + sci-libs/pgplot + sci-visualization/gnuplot + sys-apps/dog + sys-apps/eject + sys-apps/ethtool + sys-apps/hotplug + sys-apps/ipmitool + sys-apps/iproute2 + sys-apps/irqbalance + sys-apps/lm_sensors + sys-apps/parted + sys-apps/sdparm + sys-apps/slocate + sys-apps/smartmontools + sys-apps/watchdog + sys-apps/xinetd + sys-boot/grub + sys-cluster/ganglia + sys-cluster/pvm + sys-devel/bc + sys-devel/distcc + sys-devel/gdb + sys-devel/prelink + sys-fs/dosfstools + sys-fs/fuse + sys-fs/inotify-tools + sys-fs/jfsutils + sys-fs/lvm2 + sys-fs/mdadm + sys-fs/sshfs-fuse + sys-fs/xfsprogs + sys-kernel/genkernel + sys-libs/libtermcap-compat + sys-power/acpitool + sys-power/apcupsd + sys-power/pmtools + sys-process/lsof + sys-process/schedtool + sys-process/vixie-cron + www-client/links + www-client/lynx + www-servers/apache + x11-base/xorg-x11 + x11-libs/xforms + x11-misc/x11vnc + x11-misc/xsensors + x11-terms/pssh +" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.5.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.5.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-base/ipp-base-0.5.ebuild (revision 24557) @@ -0,0 +1,318 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/project/hw/portage-overlay/sys-apps/ipp-base/ipp-base-0.5.ebuild,v 1.1 2009/06/23 02:16:13 jhoblitt Exp $ + +# 2008.0 profile update + +DESCRIPTION="base set of packages for IPP compute nodes" +HOMEPAGE="pan-starrs.ifa.hawaii.edu" +SRC_URI="" + +LICENSE="" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +DEPEND="" +RDEPEND=" + app-admin/denyhosts + app-admin/logrotate + app-admin/mcelog + app-admin/showconsole + app-admin/sudo + app-admin/syslog-ng + app-admin/sysstat + app-admin/tmpwatch + app-admin/webapp-config + app-arch/cabextract + app-arch/lzma-utils + app-arch/pax + app-arch/rar + app-arch/rpm + app-arch/sharutils + app-arch/unzip + app-backup/rsnapshot + app-cdr/dvd+rw-tools + app-crypt/opencdk + app-dicts/aspell-en + app-doc/doxygen + app-editors/emacs + app-editors/gvim + app-editors/vim + app-editors/xemacs + app-emulation/emul-linux-x86-baselibs + app-emulation/emul-linux-x86-compat + app-emulation/emul-linux-x86-gtklibs + app-emulation/emul-linux-x86-qtlibs + app-emulation/emul-linux-x86-sdl + app-misc/pax-utils + app-misc/screen + app-office/openoffice-bin + app-portage/g-cpan + app-portage/gentoolkit + app-portage/layman + app-portage/mirrorselect + app-shells/bash-completion + app-shells/sash + app-shells/tcsh + app-text/aspell + app-text/sloccount + app-text/texlive + app-text/tree + app-vim/gentoo-syntax + dev-db/mysql-community + dev-java/blackdown-jdk + dev-lang/mono + dev-lang/perl + dev-lang/php + dev-lang/swig + dev-lang/tcl + dev-lang/tk + dev-libs/confuse + dev-libs/expat + dev-libs/glib + dev-libs/gmp + dev-libs/libgcrypt + dev-libs/libpcre + dev-libs/libpcre:3 + dev-libs/libxslt + dev-libs/openssl + dev-perl/Apache-DBI + dev-perl/Apache-Test + dev-perl/Array-Compare + dev-perl/Cache + dev-perl/Cache-Memcached + dev-perl/Class-Accessor + dev-perl/Class-Factory-Util + dev-perl/DBD-mysql + dev-perl/DBI + dev-perl/DateManip + dev-perl/DateTime + dev-perl/DateTime-Format-Builder + dev-perl/DateTime-Format-ISO8601 + dev-perl/DateTime-Format-Strptime + dev-perl/DateTime-Locale + dev-perl/DateTime-TimeZone + dev-perl/Digest-SHA1 + dev-perl/Email-Send + dev-perl/File-Find-Rule + dev-perl/File-NFSLock + dev-perl/File-Slurp + dev-perl/Filesys-Df + dev-perl/HTML-Parser + dev-perl/HTML-Tagset + dev-perl/Heap + dev-perl/Hook-LexWrap + dev-perl/IO-String + dev-perl/IPC-Run + dev-perl/JSON + dev-perl/Log-Log4perl + dev-perl/Net-DNS + dev-perl/Net-Domain-TLD + dev-perl/Number-Compare + dev-perl/Params-Validate + dev-perl/Parse-RecDescent + dev-perl/Proc-Daemon + dev-perl/SOAP-Lite + dev-perl/Statistics-Descriptive + dev-perl/Sub-Uplevel + dev-perl/Sys-Statistics-Linux + dev-perl/Template-Toolkit + dev-perl/Test-Exception + dev-perl/Test-Warn + dev-perl/Text-Glob + dev-perl/TimeDate + dev-perl/Tree-DAG_Node + dev-perl/URI + dev-perl/libwww-perl + dev-perl/log-dispatch + dev-perl/net-server + dev-perl/perltidy + dev-perl/yaml + dev-php/PEAR-DB + dev-python/numarray + dev-python/numeric + dev-python/numpy + dev-python/setuptools + dev-tcltk/expect + dev-util/astyle + dev-util/catalyst + dev-util/ccache + dev-util/cvs + dev-util/cvs2cl + dev-util/cvs2svn + dev-util/cvsps + dev-util/ddd + dev-util/diffstat + dev-util/efence + dev-util/git + dev-util/ltrace + dev-util/mono-tools + dev-util/oprofile + dev-util/pkgconfig + dev-util/strace + dev-util/subversion + dev-util/svk + dev-util/unifdef + dev-util/valgrind + gnome-base/gnome + gnome-base/libbonobo + mail-client/mailx + mail-client/mutt + media-fonts/corefonts + media-gfx/gimp + media-gfx/gqview + media-gfx/imagemagick + media-gfx/povray + media-gfx/xfig + media-gfx/xv + media-libs/fontconfig + media-libs/freetype + media-libs/gd + media-libs/imlib + media-libs/jasper + media-libs/jbigkit + media-libs/lcms + media-libs/libpng:1.2 + media-libs/netpbm + net-analyzer/arping + net-analyzer/gnu-netcat + net-analyzer/iftop + net-analyzer/nagios + net-analyzer/nagios-core + net-analyzer/nagios-imagepack + net-analyzer/nagios-nrpe + net-analyzer/nagios-nsca + net-analyzer/nagios-plugins + net-analyzer/net-snmp + net-analyzer/nmap + net-analyzer/tcpdump + net-analyzer/traceroute + net-dialup/minicom + net-dns/bind-tools + net-fs/autofs + net-fs/nfs-utils + net-ftp/ftp + net-ftp/ncftp + net-ftp/proftpd + net-libs/gnutls + net-misc/bridge-utils + net-misc/curl + net-misc/dhcpcd + net-misc/ifenslave + net-misc/iperf + net-misc/iputils + net-misc/memcached + net-misc/neon + net-misc/netkit-rsh + net-misc/ntp + net-misc/openssh + net-misc/rdate + net-misc/rdate + net-misc/rdesktop + net-misc/telnet-bsd + net-misc/vconfig + net-misc/vnc + net-misc/whois + net-nds/yp-tools + net-nds/ypbind + net-nds/ypserv + net-print/cups + perl-core/IPC-Cmd + perl-core/Module-Load + perl-core/Module-Load-Conditional + perl-core/Params-Check + perl-core/Test-Simple + perl-core/digest-base + perl-core/version + perl-gcpan/Abstract-Meta-Class + perl-gcpan/Astro-FITS-CFITSIO + perl-gcpan/Config-Scoped + perl-gcpan/Config-YAML + perl-gcpan/DBIx-Connection + perl-gcpan/Data-Validate-Domain + perl-gcpan/Data-Validate-URI + perl-gcpan/Digest-MD5-File + perl-gcpan/File-ExtAttr + perl-gcpan/File-Mountpoint + perl-gcpan/Net-HTTPServer + perl-gcpan/SQL-Interp + perl-gcpan/Simple-SAX-Serializer + perl-gcpan/Test-Cmd + perl-gcpan/Test-DBUnit + perl-gcpan/Test-Distribution + perl-gcpan/Test-URI + sci-libs/cfitsio + sci-libs/fftw + sci-libs/gsl + sci-libs/pgplot + sci-visualization/gnuplot + sys-apps/archttp + sys-apps/dog + sys-apps/eject + sys-apps/ethtool + sys-apps/hotplug + sys-apps/ipmitool + sys-apps/ipp-base + sys-apps/ipp-node + sys-apps/iproute2 + sys-apps/irqbalance + sys-apps/lm_sensors + sys-apps/parted + sys-apps/portage + sys-apps/sdparm + sys-apps/slocate + sys-apps/smartmontools + sys-apps/watchdog + sys-apps/xinetd + sys-boot/grub + sys-boot/syslinux + sys-cluster/ganglia + sys-cluster/pvm + sys-devel/bc + sys-devel/distcc + sys-devel/gcc-config + sys-devel/gdb + sys-devel/libperl + sys-devel/prelink + sys-fs/dosfstools + sys-fs/fuse + sys-fs/inotify-tools + sys-fs/jfsutils + sys-fs/lvm2 + sys-fs/mdadm + sys-fs/sshfs-fuse + sys-fs/xfsprogs + sys-kernel/genkernel + sys-libs/cracklib + sys-libs/libtermcap-compat + sys-libs/timezone-data + sys-power/acpitool + sys-power/apcupsd + sys-power/pmtools + sys-process/lsof + sys-process/schedtool + sys-process/vixie-cron + virtual/perl-CGI + virtual/perl-Compress-Raw-Zlib + virtual/perl-IPC-Cmd + virtual/perl-Locale-Maketext-Simple + virtual/perl-Module-Build + virtual/perl-Params-Check + virtual/perl-Storable + virtual/perl-Test-Simple + www-apache/mod_authnz_external + www-apache/mod_perl + www-apache/mod_python + www-apache/pwauth + www-apps/trac + www-client/links + www-client/lynx + www-servers/apache + x11-base/xorg-x11 + x11-libs/gtk+ + x11-libs/xforms + x11-misc/x11vnc + x11-misc/xsensors + x11-terms/pssh +" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-node/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-node/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-node/Manifest (revision 24557) @@ -0,0 +1,1 @@ +EBUILD ipp-node-0.1.ebuild 468 RMD160 4cca0d10d629b24f5175ca4e43447e112dab16be SHA1 0a1dcedeab3b232fa5b9bbb2b9f0deac65c98c3d SHA256 d8d52beea0bdab54a5f8583e04aafb26627444506c6135345b787e9d403e46b3 Index: branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-node/ipp-node-0.1.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-node/ipp-node-0.1.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/ipp-node/ipp-node-0.1.ebuild (revision 24557) @@ -0,0 +1,18 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/ipp-node/ipp-node-0.1.ebuild,v 1.2 2007/10/19 04:45:53 jhoblitt Exp $ + +DESCRIPTION="complete set of packages for an IPP cluster node" +HOMEPAGE="pan-starrs.ifa.hawaii.edu" +SRC_URI="" + +LICENSE="" +SLOT="0" +KEYWORDS="amd64" +IUSE="" + +DEPEND="" +RDEPEND=" + sys-apps/ipp-base + sys-apps/archttp +" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/ChangeLog =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/ChangeLog (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/ChangeLog (revision 24557) @@ -0,0 +1,215 @@ +# ChangeLog for sys-apps/sysvinit +# Copyright 1999-2007 Gentoo Foundation; Distributed under the GPL v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/sysvinit/ChangeLog,v 1.1 2007/07/31 21:29:15 jhoblitt Exp $ + + 11 May 2007; Joshua Kinard sysvinit-2.86-r8.ebuild: + Stable on mips. + + 30 Apr 2007; Bryan Østergaard + sysvinit-2.86-r8.ebuild: + Stable on Alpha + ia64. + + 06 Apr 2007; Peter Weller sysvinit-2.86-r7.ebuild, + sysvinit-2.86-r8.ebuild: + -r7 and -r8 stable on amd64 wrt bug 147572 + + 05 Apr 2007; Chris Gianelloni + sysvinit-2.86-r5.ebuild, sysvinit-2.86-r6.ebuild, sysvinit-2.86-r7.ebuild, + sysvinit-2.86-r8.ebuild: + Fixing the serial console for EFIKA on older revisions, and moved IBM checks + to after PPC checks so they are not overridden for ppc64 32ul. + + 27 Mar 2007; Stephen Bennett sysvinit-2.86-r7.ebuild: + mips stable + +*sysvinit-2.86-r8 (13 Mar 2007) + + 13 Mar 2007; Roy Marples files/reboot.sh, + files/shutdown.sh, +sysvinit-2.86-r8.ebuild: + Bump for non bash scripts. + + 13 Mar 2007; Raúl Porcel sysvinit-2.86-r7.ebuild: + x86 stable + + 02 Mar 2007; Brent Baude sysvinit-2.86-r7.ebuild: + Adding comment tag to sysvinit-2.86-r7 in front of the ppc serial device so + that it's added but commented out to not impact non-efiaka hw. Cleared with + Chris G. + + 26 Feb 2007; Chris Gianelloni + sysvinit-2.86-r5.ebuild, sysvinit-2.86-r6.ebuild, sysvinit-2.86-r7.ebuild: + Added EFIKA serial support if USE=ppc. + + 14 Feb 2007; Jeroen Roovers sysvinit-2.86-r7.ebuild: + Stable for HPPA (bug #158637). + + 13 Feb 2007; Chris Gianelloni + sysvinit-2.86-r6.ebuild: + Stable on alpha wrt bug #158637. + + 13 Feb 2007; Raúl Porcel sysvinit-2.86-r6.ebuild: + x86 stable wrt bug 158637 + + 13 Feb 2007; Joseph Jezak sysvinit-2.86-r7.ebuild: + Marked ppc stable. + + 06 Feb 2007; Simon Stelling sysvinit-2.86-r6.ebuild: + stable on amd64; bug 159097 + + 02 Feb 2007; Gustavo Zacarias + sysvinit-2.86-r7.ebuild: + Stable on sparc + + 11 Jan 2007; Daniel Ostrow sysvinit-2.86-r7.ebuild: + Stable on ppc64 for stage builds to include inittab again. + +*sysvinit-2.86-r7 (21 Dec 2006) + + 21 Dec 2006; Roy Marples + +files/sysvinit-2.86-shutdown-single.patch, +sysvinit-2.86-r7.ebuild: + shutdown now goes into single user mode by default, #158615. + + 12 Dec 2006; Roy Marples sysvinit-2.86-r6.ebuild: + Move sysvinit-2.86-r6 to ~ARCH. + +*sysvinit-2.86-r6 (09 Dec 2006) + + 09 Dec 2006; Mike Frysinger +files/reboot.sh, + +files/shutdown.sh, +sysvinit-2.86-r6.ebuild: + Move /etc/init.d/{reboot,shutdown}.sh to this package and disable the POSIX + caps patch for now #147572. + + 10 Sep 2006; Christel Dahlskjaer + sysvinit-2.86-r5.ebuild: + Alpha stable + + 09 Jul 2006; Joshua Kinard sysvinit-2.86-r5.ebuild: + Marked stable on mips. + + 28 Jun 2006; Luis Medinas sysvinit-2.86-r5.ebuild: + Stable on amd64. Bug #138151. + + 27 Jun 2006; Gustavo Zacarias + sysvinit-2.86-r5.ebuild: + Stable on sparc wrt #138151 + + 27 Jun 2006; Chris Gianelloni + sysvinit-2.86-r5.ebuild: + Stable on x86 wrt bug #138151. + + 27 Jun 2006; Tobias Scherbaum + sysvinit-2.86-r5.ebuild: + ppc stable, #138151 + + 27 Jun 2006; Markus Rothe sysvinit-2.86-r5.ebuild: + Stable on ppc64; bug #138151 + +*sysvinit-2.86-r5 (22 Feb 2006) + + 22 Feb 2006; Mike Frysinger +files/change_console.8, + +files/change_console.c, +sysvinit-2.86-r5.ebuild: + Add change_console utility from Fedora #123508 by Carl Anderson. + +*sysvinit-2.86-r4 (07 Feb 2006) + + 07 Feb 2006; Chris PeBenito + +files/2.86-selinux-1.patch, +sysvinit-2.86-r4.ebuild: + Bump to update SELinux patch. + +*sysvinit-2.86-r3 (20 Nov 2005) + + 20 Nov 2005; Mike Frysinger + +files/sysvinit-2.86-POSIX-1003.1e.patch, + +files/sysvinit-2.86-execl.patch, +files/sysvinit-2.86-utmp-64bit.patch, + +sysvinit-2.86-r3.ebuild: + Fix some warnings and add support for POSIX capabilites #5818 by Sascha Silbe. + +*sysvinit-2.86-r2 (28 Oct 2005) + + 28 Oct 2005; Mike Frysinger + +files/sysvinit-2.86-off-by-one.patch, +sysvinit-2.86-r2.ebuild: + Fix display bug in starting some processes #110420 by Kir Kolyshkin. + + 04 Oct 2005; Martin Schlemmer files/inittab: + Update inittab to bind agetty at least to tty1 for single mode. Noted by + RiverRat. + +*sysvinit-2.86-r1 (04 Aug 2005) + + 04 Aug 2005; Martin Schlemmer files/inittab, + +sysvinit-2.86-r1.ebuild: + Update inittab to not spawn agetty for runlevel 1 (single mode), bug #66834. + + 10 Jul 2005; Daniel Ostrow sysvinit-2.86.ebuild: + Add ibm use flag check for extra ttys on IBM hardware. + + 09 Jun 2005; Mike Frysinger sysvinit-2.86.ebuild: + Stabilize for all arches. + + 22 May 2005; Mike Frysinger sysvinit-2.86.ebuild: + Add support for kexec #80220 by Stefan Schweizer. + + 07 Jan 2005; Chris PeBenito sysvinit-2.86.ebuild: + Fix libselinux dep. The updated SELinux patch needs 1.18. + + 03 Jan 2005; Chris PeBenito + files/2.86-selinux.patch, -files/sysvinit-2.86-selinux.patch, + sysvinit-2.86.ebuild: + Fix up SELinux patch for 2.86. + +*sysvinit-2.86 (21 Dec 2004) + + 21 Dec 2004; Mike Frysinger +files/2.86-gentoo.patch, + +files/2.86-selinux.patch, files/inittab, + +files/sysvinit-2.86-selinux.patch, +sysvinit-2.86.ebuild: + Version bump #75007. + + 24 Aug 2004; Sven Wegener sysvinit-2.85-r1.ebuild: + Sync IUSE (build and bootstrap missing) + + 23 Jul 2004; Aron Griffis sysvinit-2.85-r1.ebuild: + Bring pkg_postinst over from pre-split baselayout ebuilds + +*sysvinit-2.85-r1 (09 Jul 2004) + + 09 Jul 2004; Aron Griffis +sysvinit-2.85-r1.ebuild: + Build sulogin in any case, since it's always possible to link against + /lib/libcrypt.so even when the static lib is unavailable + + 02 Jul 2004; Aron Griffis sysvinit-2.84.ebuild, + sysvinit-2.85.ebuild: + Uncomment sed line for USE=build to keep compile from failing during + stage-building + +*sysvinit-2.85 (02 Jul 2004) + + 02 Jul 2004; Chris PeBenito + +files/sysvinit-2.85-selinux.patch, +sysvinit-2.85.ebuild: + Bump, along with new SELinux patch, from #55701. + + 30 Jun 2004; Chris PeBenito sysvinit-2.84.ebuild: + Inherit eutils and fix SELinux patch versioning. + +*sysvinit-2.84 (29 Jun 2004) + + 29 Jun 2004; Aron Griffis +files/inittab, + +files/sysvinit-2.84-selinux.patch, +files/sysvinit-2.84-selinux1.patch, + +sysvinit-2.84.ebuild: + Update to version used in baselayout-1.10.1 in preparation for + baselayout/sysvinit split + +*sysvinit-2.83-r1 (09 Jul 2002) + + 10 Feb 2003; Seemant Kulleen *.ebuild : + + Changed sed expression delimiter from / to :, closing bug #15006 by Blu3 + + + 06 Dec 2002; Rodney Rees : changed sparc ~sparc keywords + + 21 Jul 2002; Mark Guertin : + updated keywords (ppc) + + 09 Jul 2002; Daniel Ahlberg ChangeLog : + + Added ChangeLog. Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/Manifest (revision 24557) @@ -0,0 +1,20 @@ +AUX 2.86-gentoo.patch 641 RMD160 9e3f9ace9e58c3352f536649491a8a5b3f89e512 SHA1 d5bd112a171285768f5618e530cab5d22f7b6f55 SHA256 6edd083b1b09b75f76b75fc299e2ddab7344a0df8f212e3cd86f27c44caa8c01 +AUX 2.86-selinux-1.patch 2628 RMD160 e17994e12ea022c5f7d8a80345fb2d3265f7044d SHA1 0057c3fec271ab8e56210e1182c5c1b5b037836a SHA256 a70ba499bdc3598fc18d0911ff32757356db353ebdd5d9f6a69aaf1ed277254f +AUX 2.86-selinux.patch 6525 RMD160 b50310333e3c7da300daadddee6135471b0a07ee SHA1 72bf861751567f47b66154102b4bbbee2e18b414 SHA256 ccf4aecf38732569bc41feaa0dcb4abc21009f8d7a21a3bc402132f5ef7f88f8 +AUX change_console.8 657 RMD160 03a46941d92dd280d92252f83f58273353c0b7bb SHA1 243fa45d631a167314b49dfc8c5a14607e44999f SHA256 918dba79243a84a3c6dfd967e44c0d7375d49cfbe0452806a981c534b6022990 +AUX change_console.c 929 RMD160 168c97e47c6dae4f378b4ae13040156fad2a75db SHA1 5fb2a3ca5faaf15ee8c9f97a169771f15fd4500c SHA256 1e61b16274dee6f27ae69db13aff9230195feb6719f650dad027520503ea69b4 +AUX inittab 1674 RMD160 322865b468f9d189ae76f671dd84bf7591ab5aef SHA1 7214fb92336b084b2c63a8fa628eda38d8452795 SHA256 60e00eb95e24f35c3ef86513f793ee3fc7e47bc839f62b12d2c27fbe5d112f27 +AUX reboot.sh 304 RMD160 bbc92c46217361abfa9ca6842e206e818959886b SHA1 2515f89492a9d225abfdd9f99fb674a8dbb66629 SHA256 967fa572014ac6dd69f5e7e24d5250abad9c20c644563b927b295778608cafef +AUX shutdown.sh 385 RMD160 5038d76195793b9996429f216fe9d4cd597b7725 SHA1 074471c08e01bfaa32bff4e6745795cf35794bef SHA256 2cc84a5194a949f16a82c6215459f4bf6d8156e50d8d9130d310f085bb208d4c +AUX sysvinit-2.86-POSIX-1003.1e.patch 1358 RMD160 a4e22a9c6c13373c0196aeb7c274e05bc0ea552f SHA1 878078b0b3c7a5090d19588085d91ba6bf749d8d SHA256 73729ddfae95826bf6f3e7f1bf0da53c8647d24126fb7a307020f5a5fa8bff04 +AUX sysvinit-2.86-docs.patch 1899 RMD160 ecc45e46d708ed3c7613e42b465f1924f91efbb8 SHA1 2cba6285beaa4c8581fe7834e640e678fc35d498 SHA256 9f79cfe861bfa146dea2c2de4849d23c82f7835b5d38a09049e9f7c1226c8808 +AUX sysvinit-2.86-execl.patch 323 RMD160 bda0a5d6acbdf32ce826295a36dd00009fe62175 SHA1 c8873f49eaf6e4566817b5773c2b93c18383f2c7 SHA256 5821fa23b8e3e14af3ce9239bfb49eadc780e58c34ca120e1d32ee3a2a1cc30b +AUX sysvinit-2.86-off-by-one.patch 310 RMD160 20d7f98ed21f4cc1c3c40d51a06f9b1a46ae3243 SHA1 191e6d43328d5edd5c68deb22e5258545700fffd SHA256 bcd48e4e9daf7abc3a824bd20ca14a922a330b15ea9ff2767ee8d20baa9b81a4 +AUX sysvinit-2.86-shutdown-single.patch 1446 RMD160 261ad2aa89b3fb5fb94d20cb5b34e9b4f1a0fa14 SHA1 50ad6630baa2c12f0e6864e8205ea10287665ea2 SHA256 f85c80fd709502681d051118a3f04089afc7367614f6ea59e32aceda7257932a +AUX sysvinit-2.86-shutdown-usage.patch 235 RMD160 644bc21c6d24ff53b34a337ce483854fe1667846 SHA1 91da19a22fe9cf093876ccc00f2e54a5995178fd SHA256 aff08a0a7c8c2188f98f71b55360bb1166b71c257b94a6e9d1a21e794b3204c7 +AUX sysvinit-2.86-utmp-64bit.patch 1112 RMD160 9ba2959e6375bee10e00ba9be9cd22d5e0103007 SHA1 da51462ad1ba256e02c435261b60f767b81440e0 SHA256 2126f4c0e1512b133da8d629691b5dc52c730e188b5790cb4e3c681e3912e03e +DIST sysvinit-2.86-kexec.patch 4139 RMD160 5f8e9ccd3a39701cf47a1a541c78a4be971d46fc SHA1 5dc952d6b1585c926a67f11eeee835e56aeea5b0 SHA256 936245ef3060939bec3099efc006009336b3672422c1de41983b277ab5ddac47 +DIST sysvinit-2.86.tar.gz 99009 RMD160 46587ab51390b5c8cd3179a3690f86efdbe9b4b9 SHA1 3e78df609a2ff0fea6304cb662a7f10298d03667 SHA256 035f98fae17d9cff002993c564ccc83dc4ed136127172caeff872b6abdb679d8 +EBUILD sysvinit-2.86-r8.ebuild 3486 RMD160 555d69bb337fae5662824d7204f69760960b062f SHA1 6678152a236d9dcf9a9485e04fa644e0c6f06766 SHA256 57b9eb5132f7e3cafc46fee7bb1a0f6d3cd8264a19f12d97a680892fc3c51d48 +MISC ChangeLog 7676 RMD160 b785ed61ed9a6935e2c5908fa868da1096e1212e SHA1 3f59ee1e7ed2c2e8736a07376f7e3eb1abecc3fb SHA256 1aba9ce922ad02e3af67b2a3dc4bc796fe2e77e263aadbca848e24ed82c1f074 +MISC metadata.xml 164 RMD160 f43cbec30b7074319087c9acffdb9354b17b0db3 SHA1 9c213f5803676c56439df3716be07d6692588856 SHA256 f5f2891f2a4791cd31350bb2bb572131ad7235cd0eeb124c9912c187ac10ce92 Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-gentoo.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-gentoo.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-gentoo.patch (revision 24557) @@ -0,0 +1,25 @@ +--- src/Makefile.orig 2004-12-21 22:08:42.607088800 -0500 ++++ src/Makefile 2004-12-21 22:12:56.409504968 -0500 +@@ -10,5 +10,3 @@ + +-CC = gcc +-CFLAGS = -Wall -O2 -fomit-frame-pointer -D_GNU_SOURCE +-LDFLAGS = -s ++CFLAGS += -Wall -D_GNU_SOURCE + STATIC = +@@ -33,2 +31,9 @@ + ++ifeq ($(DISTRO),Gentoo) ++SBIN += sulogin bootlogd ++USRBIN += utmpdump wall ++MAN1 += wall.1 ++MAN8 += sulogin.8 bootlogd.8 ++endif ++ + ifeq ($(DISTRO),Debian) +@@ -112,2 +117,5 @@ + install: ++ $(INSTALL) -d $(ROOT)/bin $(ROOT)/sbin $(ROOT)/usr/bin \ ++ $(ROOT)/usr/include $(ROOT)/$(MANDIR)/man1 $(ROOT)/$(MANDIR)/man5 \ ++ $(ROOT)/$(MANDIR)/man8 + for i in $(BIN); do \ Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-selinux-1.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-selinux-1.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-selinux-1.patch (revision 24557) @@ -0,0 +1,91 @@ +--- sysvinit-2.85/src/init.c.selinux 2005-10-14 14:16:24.000000000 -0400 ++++ sysvinit-2.85/src/init.c 2005-10-14 14:16:24.000000000 -0400 +@@ -48,6 +48,8 @@ + #include + #include + #include ++#include ++ + + #ifdef __i386__ + # if (__GLIBC__ >= 2) +@@ -2513,6 +2515,7 @@ + char *p; + int f; + int isinit; ++ int enforce = 0; + + /* Get my own name */ + if ((p = strrchr(argv[0], '/')) != NULL) +@@ -2576,6 +2579,20 @@ + maxproclen += strlen(argv[f]) + 1; + } + ++ if (getenv("SELINUX_INIT") == NULL) { ++ putenv("SELINUX_INIT=YES"); ++ if (selinux_init_load_policy(&enforce) == 0 ) { ++ execv(myname, argv); ++ } else { ++ if (enforce > 0) { ++ /* SELinux in enforcing mode but load_policy failed */ ++ /* At this point, we probably can't open /dev/console, so log() won't work */ ++ initlog(L_VB,"Unable to load SELinux Policy. Machine is in enforcing mode. Halting now."); ++ exit(1); ++ } ++ } ++ } ++ + /* Start booting. */ + argv0 = argv[0]; + argv[1] = NULL; +--- sysvinit-2.85/src/Makefile.selinux 2005-10-14 14:16:24.000000000 -0400 ++++ sysvinit-2.85/src/Makefile 2005-10-14 14:16:24.000000000 -0400 +@@ -32,7 +32,7 @@ + all: $(BIN) $(SBIN) $(USRBIN) + + init: init.o init_utmp.o +- $(CC) $(LDFLAGS) $(STATIC) -o $@ init.o init_utmp.o ++ $(CC) $(LDFLAGS) $(STATIC) -o $@ init.o init_utmp.o -lsepol -lselinux + + halt: halt.o ifdown.o hddown.o utmp.o reboot.h + $(CC) $(LDFLAGS) -o $@ halt.o ifdown.o hddown.o utmp.o +@@ -50,7 +50,7 @@ + $(CC) $(LDFLAGS) -o $@ runlevel.o + + sulogin: sulogin.o +- $(CC) $(LDFLAGS) $(STATIC) -o $@ sulogin.o $(LCRYPT) ++ $(CC) $(LDFLAGS) $(STATIC) -o $@ sulogin.o $(LCRYPT) -lselinux + + wall: dowall.o wall.o + $(CC) $(LDFLAGS) -o $@ dowall.o wall.o +--- sysvinit-2.85/src/sulogin.c.selinux 2005-10-14 14:16:24.000000000 -0400 ++++ sysvinit-2.85/src/sulogin.c 2005-10-14 14:18:42.000000000 -0400 +@@ -28,6 +28,8 @@ + # include + #endif + ++#include ++#include + + #define CHECK_DES 1 + #define CHECK_MD5 1 +@@ -332,6 +335,19 @@ + signal(SIGINT, SIG_DFL); + signal(SIGTSTP, SIG_DFL); + signal(SIGQUIT, SIG_DFL); ++ if (is_selinux_enabled > 0) { ++ security_context_t scon=NULL; ++ char *seuser=NULL; ++ char *level=NULL; ++ if (getseuserbyname("root", &seuser, &level) == 0) ++ if (get_default_context_with_level(seuser, level, 0, &scon) > 0) { ++ if (setexeccon(scon) != 0) ++ fprintf(stderr, "setexeccon failed\n"); ++ freecon(scon); ++ } ++ free(seuser); ++ free(level); ++ } + execl(sushell, shell, NULL); + perror(sushell); + Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-selinux.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-selinux.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/2.86-selinux.patch (revision 24557) @@ -0,0 +1,239 @@ +diff -urN sysvinit-2.86.orig/src/Makefile sysvinit-2.86/src/Makefile +--- sysvinit-2.86.orig/src/Makefile 2004-06-09 08:47:45.000000000 -0400 ++++ sysvinit-2.86/src/Makefile 2005-01-03 20:25:59.922659928 -0500 +@@ -12,6 +12,8 @@ + CFLAGS = -Wall -O2 -fomit-frame-pointer -D_GNU_SOURCE + LDFLAGS = -s + STATIC = ++CFLAGS += -DWITH_SELINUX ++LDFLAGS += -lselinux -lsepol + + # For some known distributions we do not build all programs, otherwise we do. + BIN = +diff -urN sysvinit-2.86.orig/src/init.c sysvinit-2.86/src/init.c +--- sysvinit-2.86.orig/src/init.c 2004-07-30 08:16:20.000000000 -0400 ++++ sysvinit-2.86/src/init.c 2005-01-03 20:46:39.485217776 -0500 +@@ -42,6 +42,11 @@ + #include + #include + #include ++#include ++#include ++#include ++#include ++ + + #ifdef __i386__ + # if (__GLIBC__ >= 2) +@@ -104,6 +109,7 @@ + int dfl_level = 0; /* Default runlevel */ + sig_atomic_t got_cont = 0; /* Set if we received the SIGCONT signal */ + sig_atomic_t got_signals; /* Set if we received a signal. */ ++int enforcing = -1; /* SELinux enforcing mode */ + int emerg_shell = 0; /* Start emergency shell? */ + int wrote_wtmp_reboot = 1; /* Set when we wrote the reboot record */ + int wrote_utmp_reboot = 1; /* Set when we wrote the reboot record */ +@@ -192,6 +198,142 @@ + char *extra_env[NR_EXTRA_ENV]; + + ++/* Mount point for selinuxfs. */ ++#define SELINUXMNT "/selinux/" ++ ++static int load_policy(int *enforce) ++{ ++ int fd=-1,ret=-1; ++ int rc=0, orig_enforce; ++ struct stat sb; ++ void *map; ++ char policy_file[PATH_MAX]; ++ int policy_version=0; ++ FILE *cfg; ++ char buf[4096]; ++ int seconfig = -2; ++ ++ selinux_getenforcemode(&seconfig); ++ ++ mount("none", "/proc", "proc", 0, 0); ++ cfg = fopen("/proc/cmdline","r"); ++ if (cfg) { ++ char *tmp; ++ if (fgets(buf,4096,cfg) && (tmp = strstr(buf,"enforcing="))) { ++ if (tmp == buf || isspace(*(tmp-1))) { ++ enforcing=atoi(tmp+10); ++ } ++ } ++ fclose(cfg); ++ } ++#define MNT_DETACH 2 ++ umount2("/proc",MNT_DETACH); ++ ++ if (enforcing >=0) ++ *enforce = enforcing; ++ else if (seconfig == 1) ++ *enforce = 1; ++ ++ if (mount("none", SELINUXMNT, "selinuxfs", 0, 0) < 0) { ++ if (errno == ENODEV) { ++ initlog(L_VB, "SELinux not supported by kernel: %s\n",SELINUXMNT,strerror(errno)); ++ *enforce = 0; ++ } else { ++ initlog(L_VB, "Failed to mount %s: %s\n",SELINUXMNT,strerror(errno)); ++ } ++ return ret; ++ } ++ ++ set_selinuxmnt(SELINUXMNT); /* set manually since we mounted it */ ++ ++ policy_version=security_policyvers(); ++ if (policy_version < 0) { ++ initlog(L_VB, "Can't get policy version: %s\n", strerror(errno)); ++ goto UMOUNT; ++ } ++ ++ orig_enforce = rc = security_getenforce(); ++ if (rc < 0) { ++ initlog(L_VB, "Can't get SELinux enforcement flag: %s\n", strerror(errno)); ++ goto UMOUNT; ++ } ++ if (enforcing >= 0) { ++ *enforce = enforcing; ++ } else if (seconfig == -1) { ++ *enforce = 0; ++ rc = security_disable(); ++ if (rc == 0) umount(SELINUXMNT); ++ if (rc < 0) { ++ rc = security_setenforce(0); ++ if (rc < 0) { ++ initlog(L_VB, "Can't disable SELinux: %s\n", strerror(errno)); ++ goto UMOUNT; ++ } ++ } ++ ret = 0; ++ goto UMOUNT; ++ } else if (seconfig >= 0) { ++ *enforce = seconfig; ++ if (orig_enforce != *enforce) { ++ rc = security_setenforce(seconfig); ++ if (rc < 0) { ++ initlog(L_VB, "Can't set SELinux enforcement flag: %s\n", strerror(errno)); ++ goto UMOUNT; ++ } ++ } ++ } ++ ++ snprintf(policy_file,sizeof(policy_file),"%s.%d",selinux_binary_policy_path(),policy_version); ++ fd = open(policy_file, O_RDONLY); ++ if (fd < 0) { ++ /* Check previous version to see if old policy is available ++ */ ++ snprintf(policy_file,sizeof(policy_file),"%s.%d",selinux_binary_policy_path(),policy_version-1); ++ fd = open(policy_file, O_RDONLY); ++ if (fd < 0) { ++ initlog(L_VB, "Can't open '%s.%d': %s\n", ++ selinux_binary_policy_path(),policy_version,strerror(errno)); ++ goto UMOUNT; ++ } ++ } ++ ++ if (fstat(fd, &sb) < 0) { ++ initlog(L_VB, "Can't stat '%s': %s\n", ++ policy_file, strerror(errno)); ++ goto UMOUNT; ++ } ++ ++ map = mmap(NULL, sb.st_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); ++ if (map == MAP_FAILED) { ++ initlog(L_VB, "Can't map '%s': %s\n", ++ policy_file, strerror(errno)); ++ goto UMOUNT; ++ } ++ ++ ++ /* Set booleans based on a booleans configuration file. */ ++ ret = sepol_genbools(map, sb.st_size, selinux_booleans_path()); ++ if (ret < 0) { ++ if (errno != ENOENT && errno != EINVAL) { ++ initlog(L_VB,"Error while setting booleans: %s\n", ++ strerror(errno)); ++ goto UMOUNT; ++ } ++ } ++ initlog(L_VB, "Loading security policy\n"); ++ ret=security_load_policy(map, sb.st_size); ++ if (ret < 0) { ++ initlog(L_VB, "security_load_policy failed\n"); ++ } ++ ++UMOUNT: ++ /*umount(SELINUXMNT); */ ++ if ( fd >= 0) { ++ close(fd); ++ } ++ return(ret); ++} ++ + /* + * Sleep a number of seconds. + * +@@ -2599,6 +2741,7 @@ + char *p; + int f; + int isinit; ++ int enforce = 0; + + /* Get my own name */ + if ((p = strrchr(argv[0], '/')) != NULL) +@@ -2662,6 +2805,20 @@ + maxproclen += strlen(argv[f]) + 1; + } + ++ if (getenv("SELINUX_INIT") == NULL) { ++ putenv("SELINUX_INIT=YES"); ++ if (load_policy(&enforce) == 0 ) { ++ execv(myname, argv); ++ } else { ++ if (enforce > 0) { ++ /* SELinux in enforcing mode but load_policy failed */ ++ /* At this point, we probably can't open /dev/console, so log() won't work */ ++ fprintf(stderr,"Enforcing mode requested but no policy loaded. Halting now.\n"); ++ exit(1); ++ } ++ } ++ } ++ + /* Start booting. */ + argv0 = argv[0]; + argv[1] = NULL; +diff -urN sysvinit-2.86.orig/src/sulogin.c sysvinit-2.86/src/sulogin.c +--- sysvinit-2.86.orig/src/sulogin.c 2004-07-30 07:40:28.000000000 -0400 ++++ sysvinit-2.86/src/sulogin.c 2005-01-03 20:30:48.746751992 -0500 +@@ -27,7 +27,10 @@ + #if defined(__GLIBC__) + # include + #endif +- ++#ifdef WITH_SELINUX ++#include ++#include ++#endif + #define CHECK_DES 1 + #define CHECK_MD5 1 + +@@ -335,6 +338,16 @@ + signal(SIGINT, SIG_DFL); + signal(SIGTSTP, SIG_DFL); + signal(SIGQUIT, SIG_DFL); ++#ifdef WITH_SELINUX ++ if (is_selinux_enabled > 0) { ++ security_context_t* contextlist=NULL; ++ if (get_ordered_context_list("root", 0, &contextlist) > 0) { ++ if (setexeccon(contextlist[0]) != 0) ++ fprintf(stderr, "setexeccon faile\n"); ++ freeconary(contextlist); ++ } ++ } ++#endif + execl(sushell, shell, NULL); + perror(sushell); + Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/change_console.8 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/change_console.8 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/change_console.8 (revision 24557) @@ -0,0 +1,33 @@ +.TH CHANGE_CONSOLE 8 "23 September 2003" "" "Linux User's Manual" + +.SH NAME +change_console -- change console used by init + +.SH SYNOPSIS +.B change_console +.RB [ \-f ] +.RB [ \-r ] +.RB [device] + +.SH DESCRIPTION +.B change_console +changes the console used by +.IR init (8) +to spawn commands. +.SH OPTIONS +.IP \fB\-f\fn +Remain in the foreground after sending the command to init +.PP +.IP \fB\-r\fn +Reset the console file used by init to its compiled-in default +(usually /dev/console). + +.IP \fBdevice\fn +The device file to use as the console. If none is specified, +the result of running ttyname on stdin is used. + +.SH SEE ALSO +.IR init (8) + +.SH AUTHOR +Red Hat, Inc. Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/change_console.c =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/change_console.c (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/change_console.c (revision 24557) @@ -0,0 +1,49 @@ +#include "initreq.h" +#include +#include +#include +#include +#include +#include +#include + +void signal_handler(int sig) { + exit(0); +} + +int main(int argc, char **argv) { + int foreground = 0, reset = 0, fd, x = 1; + char *device = NULL; + struct init_request req; + + while (x < argc) { + if (!strcmp(argv[x], "-f")) { + foreground = 1; + } else if (!strcmp(argv[x], "-r")) { + reset = 1; + } else { + device = argv[x]; + } + x++; + } + if (!device) + device = ttyname(0); + + memset(&req, 0, sizeof(req)); + req.magic = INIT_MAGIC; + req.cmd = INIT_CMD_CHANGECONS; + if (!reset) + snprintf(req.i.bsd.reserved, 127, device); + else + req.i.bsd.reserved[0] = '\0'; + signal(SIGALRM, signal_handler); + alarm(3); + if ((fd = open(INIT_FIFO, O_WRONLY)) >= 0) { + write(fd, &req, sizeof(req)) == sizeof(req); + close(fd); + } + alarm(0); + if (foreground) + pause(); + return 0; +} Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/digest-sysvinit-2.86-r8 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/digest-sysvinit-2.86-r8 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/digest-sysvinit-2.86-r8 (revision 24557) @@ -0,0 +1,6 @@ +MD5 650af823f937d386e81520b1741d4d45 sysvinit-2.86-kexec.patch 4139 +RMD160 5f8e9ccd3a39701cf47a1a541c78a4be971d46fc sysvinit-2.86-kexec.patch 4139 +SHA256 936245ef3060939bec3099efc006009336b3672422c1de41983b277ab5ddac47 sysvinit-2.86-kexec.patch 4139 +MD5 7d5d61c026122ab791ac04c8a84db967 sysvinit-2.86.tar.gz 99009 +RMD160 46587ab51390b5c8cd3179a3690f86efdbe9b4b9 sysvinit-2.86.tar.gz 99009 +SHA256 035f98fae17d9cff002993c564ccc83dc4ed136127172caeff872b6abdb679d8 sysvinit-2.86.tar.gz 99009 Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/inittab =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/inittab (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/inittab (revision 24557) @@ -0,0 +1,51 @@ +# +# /etc/inittab: This file describes how the INIT process should set up +# the system in a certain run-level. +# +# Author: Miquel van Smoorenburg, +# Modified by: Patrick J. Volkerding, +# Modified by: Daniel Robbins, +# Modified by: Martin Schlemmer, +# +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/sysvinit/files/inittab,v 1.1 2007/07/31 21:29:16 jhoblitt Exp $ + +# Default runlevel. +id:3:initdefault: + +# System initialization, mount local filesystems, etc. +si::sysinit:/sbin/rc sysinit + +# Further system initialization, brings up the boot runlevel. +rc::bootwait:/sbin/rc boot + +l0:0:wait:/sbin/rc shutdown +l1:S1:wait:/sbin/rc single +l2:2:wait:/sbin/rc nonetwork +l3:3:wait:/sbin/rc default +l4:4:wait:/sbin/rc default +l5:5:wait:/sbin/rc default +l6:6:wait:/sbin/rc reboot +#z6:6:respawn:/sbin/sulogin + +# TERMINALS +c1:12345:respawn:/sbin/agetty 38400 tty1 linux +c2:2345:respawn:/sbin/agetty 38400 tty2 linux +c3:2345:respawn:/sbin/agetty 38400 tty3 linux +c4:2345:respawn:/sbin/agetty 38400 tty4 linux +c5:2345:respawn:/sbin/agetty 38400 tty5 linux +c6:2345:respawn:/sbin/agetty 38400 tty6 linux + +# SERIAL CONSOLES +#s0:12345:respawn:/sbin/agetty 9600 ttyS0 vt100 +#s1:12345:respawn:/sbin/agetty 9600 ttyS1 vt100 + +# What to do at the "Three Finger Salute". +ca:12345:ctrlaltdel:/sbin/shutdown -r now + +# Used by /etc/init.d/xdm to control DM startup. +# Read the comments in /etc/init.d/xdm for more +# info. Do NOT remove, as this will start nothing +# extra at boot if /etc/init.d/xdm is not added +# to the "default" runlevel. +x:a:once:/etc/X11/startDM.sh + Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/reboot.sh =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/reboot.sh (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/reboot.sh (revision 24557) @@ -0,0 +1,11 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +opts="-dpk" +[ "${RC_DOWN_INTERFACE}" = "yes" ] && opts="${opts}i" + +/sbin/reboot "${opts}" 2>/dev/null + +# hmm, if the above failed, that's kind of odd ... +# so let's force a reboot +/sbin/reboot -f Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/shutdown.sh =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/shutdown.sh (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/shutdown.sh (revision 24557) @@ -0,0 +1,13 @@ +# Copyright 1999-2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +opts="-d" +[ "${INIT_HALT}" != "HALT" ] && opts="${opts}p" +[ "${RC_DOWN_INTERFACE}" = "yes" ] && opts="${opts}i" +[ "${RC_DOWN_HARDDISK}" = "yes" ] && opts="${opts}h" + +/sbin/halt "${opts}" + +# hmm, if the above failed, that's kind of odd ... +# so let's force a halt +/sbin/halt -f Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-POSIX-1003.1e.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-POSIX-1003.1e.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-POSIX-1003.1e.patch (revision 24557) @@ -0,0 +1,67 @@ +add support for POSIX capabilites + +http://original.killa.net/infosec/caps/ +http://bugs.gentoo.org/5818 + +--- sysvinit/src/init.c ++++ sysvinit/src/init.c +@@ -21,6 +21,15 @@ + */ + + #include ++ ++#ifdef __linux__ ++#include ++#include ++#include ++#define capget(header, data) syscall(__NR_capget, header, data) ++#define capset(header, data) syscall(__NR_capset, header, data) ++#endif ++ + #include + #include + #include +@@ -2348,6 +2357,11 @@ int init_main() + pid_t rc; + int f, st; + ++#ifdef __linux__ ++ cap_user_header_t head; ++ cap_user_data_t data; ++#endif ++ + if (!reload) { + + #if INITDEBUG +@@ -2438,6 +2452,31 @@ int init_main() + SETSIG(sa, SIGCHLD, chld_handler, SA_RESTART); + } + ++#ifdef __linux__ ++ /* ++ * This seems like the right place to do this, just before ++ * we read /etc/inittab... ++ */ ++ ++ head = malloc(sizeof(head)); ++ if ((data = malloc(sizeof(data))) == NULL) ++ free(head); ++ ++ if (head && data) { ++ head->pid = 0; ++ head->version = _LINUX_CAPABILITY_VERSION; ++ ++ if (capget(head, data) == 0) { ++ /* Max out the inheritable capability set. */ ++ data->inheritable = data->effective; ++ capset(head, data); ++ } ++ ++ free(head); ++ free(data); ++ } ++#endif ++ + /* + * Start normal boot procedure. + */ Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-docs.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-docs.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-docs.patch (revision 24557) @@ -0,0 +1,73 @@ +Sync both the content and style of `shutdown -h` and `man shutdown`. + +http://bugs.gentoo.org/91724 + +--- man/shutdown.8 ++++ man/shutdown.8 +@@ -11,7 +11,7 @@ + .B /sbin/shutdown ++.RB [ \-akrhPHfFnc ] + .RB [ \-t + .IR sec ] +-.RB [ \-arkhncfFHP ] + .I time +-.RI [ warning-message ] ++.RI [ "warning message" ] + .\"}}} +@@ -41,7 +41,2 @@ + .\"}}} +-.\"{{{ -t sec +-.IP "\fB\-t\fP \fIsec\fP" +-Tell \fBinit\fP(8) to wait \fIsec\fP seconds between sending processes the +-warning and the kill signal, before changing to another runlevel. +-.\"}}} + .\"{{{ -k +@@ -58,7 +53,2 @@ + .\"}}} +-.\"{{{ -H +-.IP \fB\-H\fP +-Halt action is to halt or drop into boot monitor on systems that +-support it. +-.\"}}} + .\"{{{ -P +@@ -67,7 +57,6 @@ + .\"}}} +-.\"{{{ -n +-.IP \fB\-n\fP +-[DEPRECATED] Don't call \fBinit\fP(8) to do the shutdown but do it ourself. +-The use of this option is discouraged, and its results are not always what +-you'd expect. ++.\"{{{ -H ++.IP \fB\-H\fP ++Halt action is to halt or drop into boot monitor on systems that ++support it. + .\"}}} +@@ -81,2 +70,8 @@ + .\"}}} ++.\"{{{ -n ++.IP \fB\-n\fP ++[DEPRECATED] Don't call \fBinit\fP(8) to do the shutdown but do it ourself. ++The use of this option is discouraged, and its results are not always what ++you'd expect. ++.\"}}} + .\"{{{ -c +@@ -87,2 +82,7 @@ + .\"}}} ++.\"{{{ -t sec ++.IP "\fB\-t\fP \fIsec\fP" ++Tell \fBinit\fP(8) to wait \fIsec\fP seconds between sending processes the ++warning and the kill signal, before changing to another runlevel. ++.\"}}} + .\"{{{ time +@@ -92,3 +92,3 @@ + .\"{{{ warning-message +-.IP \fIwarning-message\fP ++.IP "\fIwarning message\fP" + Message to send to all users. +--- src/shutdown.c ++++ src/shutdown.c +@@ -104,3 +104,3 @@ + fprintf(stderr, +- "Usage:\t shutdown [-akrhHPfnc] [-t secs] time [warning message]\n" ++ "Usage:\t shutdown [-akrhPHfFnc] [-t sec] time [warning message]\n" + "\t\t -a: use /etc/shutdown.allow\n" Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-execl.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-execl.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-execl.patch (revision 24557) @@ -0,0 +1,11 @@ +--- sysvinit/src/init.c ++++ sysvinit/src/init.c +@@ -1893,7 +1893,7 @@ void re_exec(void) + * The existing init process execs a new init binary. + */ + env = init_buildenv(0); +- execl(myname, myname, "--init", NULL, env); ++ execle(myname, myname, "--init", NULL, env); + + /* + * We shouldn't be here, something failed. Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-off-by-one.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-off-by-one.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-off-by-one.patch (revision 24557) @@ -0,0 +1,16 @@ +Fix bug where proc names are truncated + +http://bugzilla.openvz.org/60 +http://bugs.gentoo.org/110420 + +--- src/init.c ++++ src/init.c +@@ -466,7 +466,7 @@ + + if (maxproclen > 2) { + memset(argv0, 0, maxproclen); +- strncpy(argv0, buf, maxproclen - 2); ++ strncpy(argv0, buf, maxproclen - 1); + } + + return len; Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-shutdown-single.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-shutdown-single.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-shutdown-single.patch (revision 24557) @@ -0,0 +1,38 @@ +This patch makes shutdown use the Single User runlevel by default so +the gettys are shutdown too. + +Roy Marples + +diff -ur sysvinit-2.86.orig/man/shutdown.8 sysvinit-2.86/man/shutdown.8 +--- sysvinit-2.86.orig/man/shutdown.8 2006-12-21 10:16:37.000000000 +0000 ++++ sysvinit-2.86/man/shutdown.8 2006-12-21 10:19:51.000000000 +0000 +@@ -27,7 +27,7 @@ + \fBshutdown\fP does its job by signalling the \fBinit\fP process, + asking it to change the runlevel. + Runlevel \fB0\fP is used to halt the system, runlevel \fB6\fP is used +-to reboot the system, and runlevel \fB1\fP is used to put to system into ++to reboot the system, and runlevel \fBS\fP is used to put to system into + a state where administrative tasks can be performed; this is the default + if neither the \fI-h\fP or \fI-r\fP flag is given to \fBshutdown\fP. + To see which actions are taken on halt or reboot see the appropriate +diff -ur sysvinit-2.86.orig/src/shutdown.c sysvinit-2.86/src/shutdown.c +--- sysvinit-2.86.orig/src/shutdown.c 2006-12-21 10:16:37.000000000 +0000 ++++ sysvinit-2.86/src/shutdown.c 2006-12-21 10:21:03.000000000 +0000 +@@ -463,7 +463,7 @@ + usage(); + exit(1); + } +- strcpy(down_level, "1"); ++ strcpy(down_level, "S"); + halttype = NULL; + + /* Process the options. */ +@@ -636,6 +636,8 @@ + strcpy(newstate, "for reboot"); + break; + case '1': ++ case 'S': ++ case 's': + strcpy(newstate, "to maintenance mode"); + break; + default: Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-shutdown-usage.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-shutdown-usage.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-shutdown-usage.patch (revision 24557) @@ -0,0 +1,10 @@ +Allow non-root users to at least see the help output. + +http://bugs.gentoo.org/91724 + +--- src/shutdown.c ++++ src/shutdown.c +@@ -462,2 +462,3 @@ + fprintf(stderr, "shutdown: you must be root to do that!\n"); ++ usage(); + exit(1); Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-utmp-64bit.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-utmp-64bit.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/files/sysvinit-2.86-utmp-64bit.patch (revision 24557) @@ -0,0 +1,43 @@ +On 64bit arches which have 32bit multilib support, the utmp struct is +the same ... thus we cannot assume that utmp.ut_tv is of type time_t + +--- sysvinit/src/utmp.c ++++ sysvinit/src/utmp.c +@@ -47,6 +47,7 @@ void write_wtmp + int fd; + struct utmp utmp; + struct utsname uname_buf; ++ struct timeval tv; + + /* + * Try to open the wtmp file. Note that we even try +@@ -76,7 +77,9 @@ void write_wtmp + */ + memset(&utmp, 0, sizeof(utmp)); + #if defined(__GLIBC__) +- gettimeofday(&utmp.ut_tv, NULL); ++ gettimeofday(&tv, NULL); ++ utmp.ut_tv.tv_sec = tv.tv_sec; ++ utmp.ut_tv.tv_usec = tv.tv_usec; + #else + time(&utmp.ut_time); + #endif +@@ -113,6 +116,7 @@ static void write_utmp + struct utmp utmp; + struct utmp tmp; + struct utmp *utmptr; ++ struct timeval tv; + + /* + * Can't do much if UTMP_FILE is not present. +@@ -144,7 +148,9 @@ static void write_utmp + utmp.ut_pid = pid; + strncpy(utmp.ut_id, id, sizeof(utmp.ut_id)); + #if defined(__GLIBC__) +- gettimeofday(&utmp.ut_tv, NULL); ++ gettimeofday(&tv, NULL); ++ utmp.ut_tv.tv_sec = tv.tv_sec; ++ utmp.ut_tv.tv_usec = tv.tv_usec; + #else + time(&utmp.ut_time); + #endif Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/metadata.xml =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/metadata.xml (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/metadata.xml (revision 24557) @@ -0,0 +1,5 @@ + + + +base-system + Index: branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/sysvinit-2.86-r8.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/sysvinit-2.86-r8.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-apps/sysvinit/sysvinit-2.86-r8.ebuild (revision 24557) @@ -0,0 +1,98 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/datasys/IPP/hw/portage-overlay/sys-apps/sysvinit/sysvinit-2.86-r8.ebuild,v 1.1 2007/07/31 21:29:15 jhoblitt Exp $ + +inherit eutils toolchain-funcs flag-o-matic + +DESCRIPTION="/sbin/init - parent of all processes" +HOMEPAGE="http://freshmeat.net/projects/sysvinit/" +SRC_URI="ftp://ftp.cistron.nl/pub/people/miquels/software/${P}.tar.gz + ftp://sunsite.unc.edu/pub/Linux/system/daemons/init/${P}.tar.gz + http://www.gc-linux.org/down/isobel/kexec/sysvinit/sysvinit-2.86-kexec.patch" + +LICENSE="GPL-2" +SLOT="0" +KEYWORDS="alpha amd64 arm hppa ia64 m68k mips ppc ppc64 s390 sh sparc x86" +IUSE="selinux ibm static" + +RDEPEND="selinux? ( >=sys-libs/libselinux-1.28 )" +DEPEND="${RDEPEND} + virtual/os-headers" + +src_unpack() { + unpack ${P}.tar.gz + cd "${S}" + cp "${FILESDIR}"/change_console.{c,8} src/ || die + epatch "${FILESDIR}"/${P}-docs.patch + epatch "${FILESDIR}"/${P}-shutdown-usage.patch + epatch "${FILESDIR}"/sysvinit-2.86-off-by-one.patch + epatch "${DISTDIR}"/sysvinit-2.86-kexec.patch + #epatch "${FILESDIR}"/sysvinit-2.86-POSIX-1003.1e.patch #5818 + epatch "${FILESDIR}"/sysvinit-2.86-execl.patch + epatch "${FILESDIR}"/sysvinit-2.86-utmp-64bit.patch + epatch "${FILESDIR}/${P}"-shutdown-single.patch + cd src + epatch "${FILESDIR}"/${PV}-gentoo.patch + use selinux && epatch "${FILESDIR}"/${PV}-selinux-1.patch + + # Mung inittab for specific architectures + cd "${WORKDIR}" + cp "${FILESDIR}"/inittab . || die "cp inittab" + local insert="" + use amd64 && insert="s0:12345:respawn:/sbin/agetty 115200 ttyS0 vt100" + use ppc && insert="#psc0:12345:respawn:/sbin/agetty 115200 ttyPSC0 linux\n" + use arm && insert='#f0:12345:respawn:/sbin/agetty 9600 ttyFB0 vt100' + use hppa && insert='b0:12345:respawn:/sbin/agetty 9600 ttyB0 vt100' + use s390 && insert='s0:12345:respawn:/sbin/agetty 38400 console' + if use ibm ; then + insert="${insert}#hvc0:2345:respawn:/sbin/agetty -L 9600 hvc0"$'\n' + insert="${insert}#hvsi:2345:respawn:/sbin/agetty -L 19200 hvsi0" + fi + (use arm || use mips || use sh || use sparc) && sed -i '/ttyS0/s:#::' inittab + [[ -n ${insert} ]] && echo "# Architecture specific features"$'\n'"${insert}" >> inittab +} + +src_compile() { + use static && append-ldflags -static + + # Note: The LCRYPT define below overrides the test in + # sysvinit's Makefile. This is because sulogin must be linked + # to libcrypt in any case, but when building stage2 in + # catalyst, /usr/lib/libcrypt.a isn't available. In truth + # this doesn't change how sulogin is built since ld would use + # the shared obj by default anyway! The other option is to + # refrain from building sulogin, but that isn't a good option. + # (09 Jul 2004 agriffis) + emake -C src \ + CC="$(tc-getCC)" \ + DISTRO="Gentoo" \ + LCRYPT="-lcrypt" \ + all change_console \ + || die +} + +src_install() { + dodoc README doc/* + + cd src + make install DISTRO="Gentoo" ROOT="${D}" || die "make install" + + into / + dosbin change_console || die + doman change_console.8 + + insinto /etc + doins "${WORKDIR}"/inittab || die "inittab" + + doinitd "${FILESDIR}"/{reboot,shutdown}.sh || die +} + +pkg_postinst() { + # Reload init to fix unmounting problems of / on next reboot. + # This is really needed, as without the new version of init cause init + # not to quit properly on reboot, and causes a fsck of / on next reboot. + if [[ ${ROOT} == / ]] ; then + # Do not return an error if this fails + /sbin/telinit U &>/dev/null + fi +} Index: branches/eam_branches/20090522/portage_overlay/sys-boot/grub/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-boot/grub/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-boot/grub/Manifest (revision 24557) @@ -0,0 +1,5 @@ +AUX grub-gpt.patch 10682 RMD160 53dd38d52f2b38fb28dfb670d6fd27e47a562bbb SHA1 6057a076110d469fb63e56e57711d281221f9b42 SHA256 5e38046cfb727420e5b4b19d71c7fb4ddff22cbcc356c9f888763b0c1963d389 +DIST grub-0.97-patches-1.4.tar.bz2 24693 RMD160 6afefff73b5f3ab5c6d05563c4f67f37dbfce16c SHA1 7c26a941467b58380747c9d1e07c14dfc3998b10 SHA256 290dd96fdeb9471c516f061f4949ccda33f8d03ab81a6e2beb5005b3022e71da +DIST grub-0.97.tar.gz 971783 RMD160 7fb5674edf0c950bd38e94f85ff1e2909aa741f0 SHA1 2580626c4579bd99336d3af4482c346c95dac4fb SHA256 4e1d15d12dbd3e9208111d6b806ad5a9857ca8850c47877d36575b904559260b +DIST splash.xpm.gz 33856 RMD160 2fead61f91c368429e80936248bb02af2bdf15ff SHA1 98e679f9698be43426f971f89a088c053e8c804a SHA256 b95600f777331b0dd31d51c68f60f0e846e4c8b628857a41165f4e6b30e6acaf +EBUILD grub-0.97-r3.ebuild 4688 RMD160 aa773d6e3cb6ab3683d712c85c758cb3a18d561e SHA1 fe475dfd720ed756fcf1b43fb6c1a7c32c57fca7 SHA256 260faca8d4b18028616293c380071ba14e525fb98fe48229737ed5f410150e54 Index: branches/eam_branches/20090522/portage_overlay/sys-boot/grub/files/digest-grub-0.97-r3 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-boot/grub/files/digest-grub-0.97-r3 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-boot/grub/files/digest-grub-0.97-r3 (revision 24557) @@ -0,0 +1,9 @@ +MD5 995985c7bd6d9680b36d6efd677fb751 grub-0.97-patches-1.4.tar.bz2 24693 +RMD160 6afefff73b5f3ab5c6d05563c4f67f37dbfce16c grub-0.97-patches-1.4.tar.bz2 24693 +SHA256 290dd96fdeb9471c516f061f4949ccda33f8d03ab81a6e2beb5005b3022e71da grub-0.97-patches-1.4.tar.bz2 24693 +MD5 cd3f3eb54446be6003156158d51f4884 grub-0.97.tar.gz 971783 +RMD160 7fb5674edf0c950bd38e94f85ff1e2909aa741f0 grub-0.97.tar.gz 971783 +SHA256 4e1d15d12dbd3e9208111d6b806ad5a9857ca8850c47877d36575b904559260b grub-0.97.tar.gz 971783 +MD5 cdd6c89d48982ecd544c0b4774755afd splash.xpm.gz 33856 +RMD160 2fead61f91c368429e80936248bb02af2bdf15ff splash.xpm.gz 33856 +SHA256 b95600f777331b0dd31d51c68f60f0e846e4c8b628857a41165f4e6b30e6acaf splash.xpm.gz 33856 Index: branches/eam_branches/20090522/portage_overlay/sys-boot/grub/files/grub-gpt.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-boot/grub/files/grub-gpt.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-boot/grub/files/grub-gpt.patch (revision 24557) @@ -0,0 +1,315 @@ +diff -ruBbd --unidirectional-new-file grub-0.96/stage2/builtins.c grub-0.96-patched/stage2/builtins.c +--- grub-0.96/stage2/builtins.c 2004-06-20 09:33:04.000000000 -0400 ++++ grub-0.96-patched/stage2/builtins.c 2007-01-04 13:56:06.000000000 -0500 +@@ -1229,14 +1229,15 @@ + for (drive = 0x80; drive < 0x88; drive++) + { + unsigned long part = 0xFFFFFF; +- unsigned long start, len, offset, ext_offset; +- int type, entry; ++ unsigned long start, len, offset, ext_offset, gpt_offset; ++ int type, entry, gpt_count, gpt_size; + char buf[SECTOR_SIZE]; + + current_drive = drive; + while (next_partition (drive, 0xFFFFFF, &part, &type, + &start, &len, &offset, &entry, +- &ext_offset, buf)) ++ &ext_offset, &gpt_offset, ++ &gpt_count, &gpt_size, buf)) + { + if (type != PC_SLICE_TYPE_NONE + && ! IS_PC_SLICE_TYPE_BSD (type) +@@ -2806,8 +2807,8 @@ + { + int new_type; + unsigned long part = 0xFFFFFF; +- unsigned long start, len, offset, ext_offset; +- int entry, type; ++ unsigned long start, len, offset, ext_offset, gpt_offset; ++ int entry, type, gpt_count, gpt_size; + char mbr[512]; + + /* Get the drive and the partition. */ +@@ -2844,7 +2845,14 @@ + /* Look for the partition. */ + while (next_partition (current_drive, 0xFFFFFF, &part, &type, + &start, &len, &offset, &entry, +- &ext_offset, mbr)) ++ &ext_offset, &gpt_offset, &gpt_count, &gpt_size, mbr)) ++ /* The partition may not be a GPT partition. */ ++ if (gpt_offset != 0) ++ { ++ errnum = ERR_BAD_ARGUMENT; ++ return 1; ++ } ++ + { + if (part == current_partition) + { +diff -ruBbd --unidirectional-new-file grub-0.96/stage2/disk_io.c grub-0.96-patched/stage2/disk_io.c +--- grub-0.96/stage2/disk_io.c 2004-05-23 12:35:24.000000000 -0400 ++++ grub-0.96-patched/stage2/disk_io.c 2007-01-04 14:01:08.000000000 -0500 +@@ -21,6 +21,7 @@ + + #include + #include ++#include + + #ifdef SUPPORT_NETBOOT + # define GRUB 1 +@@ -502,8 +503,8 @@ + set_partition_hidden_flag (int hidden) + { + unsigned long part = 0xFFFFFF; +- unsigned long start, len, offset, ext_offset; +- int entry, type; ++ unsigned long start, len, offset, ext_offset, gpt_offset; ++ int entry, type, gpt_count, gpt_size; + char mbr[512]; + + /* The drive must be a hard disk. */ +@@ -524,7 +525,14 @@ + /* Look for the partition. */ + while (next_partition (current_drive, 0xFFFFFF, &part, &type, + &start, &len, &offset, &entry, +- &ext_offset, mbr)) ++ &ext_offset, &gpt_offset, &gpt_count, &gpt_size, mbr)) ++ /* The partition may not be a GPT partition. */ ++ if (gpt_offset != 0) ++ { ++ errnum = ERR_BAD_ARGUMENT; ++ return 1; ++ } ++ + { + if (part == current_partition) + { +@@ -577,11 +585,14 @@ + unsigned long *partition, int *type, + unsigned long *start, unsigned long *len, + unsigned long *offset, int *entry, +- unsigned long *ext_offset, char *buf) ++ unsigned long *ext_offset, ++ unsigned long *gpt_offset, int *gpt_count, ++ int *gpt_size, char *buf) + { + /* Forward declarations. */ + auto int next_bsd_partition (void); + auto int next_pc_slice (void); ++ auto int next_gpt_slice(void); + + /* Get next BSD partition in current PC slice. */ + int next_bsd_partition (void) +@@ -666,6 +677,40 @@ + return 0; + } + ++ /* If this is a GPT partition table, read it as such. */ ++ if (*entry == -1 && *offset == 0 && PC_SLICE_TYPE (buf, 0) == PC_SLICE_TYPE_GPT) ++ { ++ struct grub_gpt_header *hdr = (struct grub_gpt_header *) buf; ++ ++ /* Read in the GPT Partition table header. */ ++ if (! rawread (drive, 1, 0, SECTOR_SIZE, buf)) ++ return 0; ++ ++ if (hdr->magic == GPT_HEADER_MAGIC && hdr->version == 0x10000) ++ { ++ /* Let gpt_offset point to the first entry in the GPT ++ partition table. This can also be used by callers of ++ next_partition to determine if a entry comes from a ++ GPT partition table or not. */ ++ *gpt_offset = hdr->partitions; ++ *gpt_count = hdr->maxpart; ++ *gpt_size = hdr->partentry_size; ++ ++ return next_gpt_slice(); ++ } ++ else ++ { ++ /* This is not a valid header for a GPT partition table. ++ Re-read the MBR or the boot sector of the extended ++ partition. */ ++ if (! rawread (drive, *offset, 0, SECTOR_SIZE, buf)) ++ return 0; ++ } ++ } ++ ++ /* Not a GPT partition. */ ++ *gpt_offset = 0; ++ + /* Increase the entry number. */ + (*entry)++; + +@@ -710,6 +755,43 @@ + return 1; + } + ++ /* Get the next GPT slice. */ ++ int next_gpt_slice (void) ++ { ++ struct grub_gpt_partentry *gptentry = (struct grub_gpt_partentry *) buf; ++ /* Make GPT partitions show up as PC slices. */ ++ int pc_slice_no = (*partition & 0xFF0000) >> 16; ++ ++ /* If this is the first time... */ ++ if (pc_slice_no == 0xFF) ++ { ++ pc_slice_no = -1; ++ *entry = -1; ++ } ++ ++ do { ++ (*entry)++; ++ ++ if (*entry >= *gpt_count) ++ { ++ errnum = ERR_NO_PART; ++ return 0; ++ } ++ /* Read in the GPT Partition table entry. */ ++ if (! rawread (drive, (*gpt_offset) + GPT_ENTRY_SECTOR (*gpt_size, *entry), GPT_ENTRY_INDEX (*gpt_size, *entry), *gpt_size, buf)) ++ return 0; ++ } while (! (gptentry->type1 && gptentry->type2)); ++ ++ pc_slice_no++; ++ *start = gptentry->start; ++ *len = gptentry->end - gptentry->start + 1; ++ *type = PC_SLICE_TYPE_EXT2FS; ++ *entry = pc_slice_no; ++ *partition = (*entry << 16) | 0xFFFF; ++ ++ return 1; ++ } ++ + /* Start the body of this function. */ + + #ifndef STAGE1_5 +@@ -717,6 +799,9 @@ + return 0; + #endif + ++ if (*partition != 0xFFFFFF && *gpt_offset != 0) ++ return next_gpt_slice (); ++ + /* If previous partition is a BSD partition or a PC slice which + contains BSD partitions... */ + if ((*partition != 0xFFFFFF && IS_PC_SLICE_TYPE_BSD (*type & 0xff)) +@@ -755,6 +840,9 @@ + unsigned long dest_partition = current_partition; + unsigned long part_offset; + unsigned long ext_offset; ++ unsigned long gpt_offset; ++ int gpt_count; ++ int gpt_size; + int entry; + char buf[SECTOR_SIZE]; + int bsd_part, pc_slice; +@@ -766,7 +854,8 @@ + int ret = next_partition (current_drive, dest_partition, + ¤t_partition, ¤t_slice, + &part_start, &part_length, +- &part_offset, &entry, &ext_offset, buf); ++ &part_offset, &entry, &ext_offset, ++ &gpt_offset, &gpt_count, &gpt_size, buf); + bsd_part = (current_partition >> 8) & 0xFF; + pc_slice = current_partition >> 16; + return ret; +diff -ruBbd --unidirectional-new-file grub-0.96/stage2/gpt.h grub-0.96-patched/stage2/gpt.h +--- grub-0.96/stage2/gpt.h 1969-12-31 19:00:00.000000000 -0500 ++++ grub-0.96-patched/stage2/gpt.h 2007-01-04 13:52:14.000000000 -0500 +@@ -0,0 +1,68 @@ ++/* ++ * GRUB -- GRand Unified Bootloader ++ * Copyright (C) 2002,2005,2006 Free Software Foundation, Inc. ++ * ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2 of the License, or ++ * (at your option) any later version. ++ * ++ * This program is distributed in the hope that it will be useful, ++ * but WITHOUT ANY WARRANTY; without even the implied warranty of ++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++ * GNU General Public License for more details. ++ * ++ * You should have received a copy of the GNU General Public License ++ * along with this program; if not, write to the Free Software ++ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ++ */ ++ ++#ifndef _GPT_H ++#define _GPT_H ++ ++typedef signed char grub_int8_t; ++typedef signed short grub_int16_t; ++typedef signed int grub_int32_t; ++typedef signed long long int grub_int64_t; ++typedef unsigned char grub_uint8_t; ++typedef unsigned short grub_uint16_t; ++typedef unsigned int grub_uint32_t; ++typedef unsigned long long int grub_uint64_t; ++ ++struct grub_gpt_header ++{ ++ grub_uint64_t magic; ++ grub_uint32_t version; ++ grub_uint32_t headersize; ++ grub_uint32_t crc32; ++ grub_uint32_t unused1; ++ grub_uint64_t primary; ++ grub_uint64_t backup; ++ grub_uint64_t start; ++ grub_uint64_t end; ++ grub_uint8_t guid[16]; ++ grub_uint64_t partitions; ++ grub_uint32_t maxpart; ++ grub_uint32_t partentry_size; ++ grub_uint32_t partentry_crc32; ++} __attribute__ ((packed)); ++ ++struct grub_gpt_partentry ++{ ++ grub_uint64_t type1; ++ grub_uint64_t type2; ++ grub_uint8_t guid[16]; ++ grub_uint64_t start; ++ grub_uint64_t end; ++ grub_uint8_t attrib; ++ char name[72]; ++} __attribute__ ((packed)); ++ ++#define GPT_HEADER_MAGIC 0x5452415020494645UL ++ ++#define GPT_ENTRY_SECTOR(size,entry) \ ++ ((((entry) * (size) + 1) & ~(SECTOR_SIZE - 1)) >> SECTOR_BITS) ++#define GPT_ENTRY_INDEX(size,entry) \ ++ ((((entry) * (size) + 1) & (SECTOR_SIZE - 1)) - 1) ++ ++#endif /* _GPT_H */ +diff -ruBbd --unidirectional-new-file grub-0.96/stage2/pc_slice.h grub-0.96-patched/stage2/pc_slice.h +--- grub-0.96/stage2/pc_slice.h 2003-07-09 07:45:53.000000000 -0400 ++++ grub-0.96-patched/stage2/pc_slice.h 2007-01-04 13:52:14.000000000 -0500 +@@ -115,6 +115,7 @@ + #define PC_SLICE_TYPE_LINUX_EXTENDED 0x85 + #define PC_SLICE_TYPE_VSTAFS 0x9e + #define PC_SLICE_TYPE_DELL_UTIL 0xde ++#define PC_SLICE_TYPE_GPT 0xee + #define PC_SLICE_TYPE_LINUX_RAID 0xfd + + +diff -ruBbd --unidirectional-new-file grub-0.96/stage2/shared.h grub-0.96-patched/stage2/shared.h +--- grub-0.96/stage2/shared.h 2004-06-19 12:40:09.000000000 -0400 ++++ grub-0.96-patched/stage2/shared.h 2007-01-04 13:52:15.000000000 -0500 +@@ -934,7 +934,9 @@ + unsigned long *partition, int *type, + unsigned long *start, unsigned long *len, + unsigned long *offset, int *entry, +- unsigned long *ext_offset, char *buf); ++ unsigned long *ext_offset, ++ unsigned long *gpt_offset, int *gpt_count, ++ int *gpt_size, char *buf); + + /* Sets device to the one represented by the SAVED_* parameters. */ + int make_saved_active (void); Index: branches/eam_branches/20090522/portage_overlay/sys-boot/grub/grub-0.97-r3.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-boot/grub/grub-0.97-r3.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-boot/grub/grub-0.97-r3.ebuild (revision 24557) @@ -0,0 +1,163 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /var/cvsroot/gentoo-x86/sys-boot/grub/grub-0.97-r3.ebuild,v 1.8 2007/03/25 12:36:50 vapier Exp $ + +inherit mount-boot eutils flag-o-matic toolchain-funcs autotools + +PATCHVER="1.4" +DESCRIPTION="GNU GRUB Legacy boot loader" +HOMEPAGE="http://www.gnu.org/software/grub/" +SRC_URI="mirror://gentoo/${P}.tar.gz + ftp://alpha.gnu.org/gnu/${PN}/${P}.tar.gz + mirror://gentoo/splash.xpm.gz + mirror://gentoo/${P}-patches-${PATCHVER}.tar.bz2" + +LICENSE="GPL-2" +SLOT="0" +KEYWORDS="amd64 x86" +IUSE="static netboot custom-cflags" + +DEPEND=">=sys-libs/ncurses-5.2-r5" +PROVIDE="virtual/bootloader" + +src_unpack() { + unpack ${A} + cd "${S}" + + # patch breaks booting for some people #111885 + rm "${WORKDIR}"/patch/400_* + + epatch "${FILESDIR}"/grub-gpt.patch + + if [[ -n ${PATCHVER} ]] ; then + EPATCH_SUFFIX="patch" + epatch "${WORKDIR}"/patch + eautoreconf + fi +} + +src_compile() { + filter-flags -fPIE #168834 + + use amd64 && multilib_toolchain_setup x86 + + unset BLOCK_SIZE #73499 + + ### i686-specific code in the boot loader is a bad idea; disabling to ensure + ### at least some compatibility if the hard drive is moved to an older or + ### incompatible system. + + # grub-0.95 added -fno-stack-protector detection, to disable ssp for stage2, + # but the objcopy's (faulty) test fails if -fstack-protector is default. + # create a cache telling configure that objcopy is ok, and add -C to econf + # to make use of the cache. + # + # CFLAGS has to be undefined running econf, else -fno-stack-protector detection fails. + # STAGE2_CFLAGS is not allowed to be used on emake command-line, it overwrites + # -fno-stack-protector detected by configure, removed from netboot's emake. + use custom-cflags || unset CFLAGS + + export grub_cv_prog_objcopy_absolute=yes #79734 + use static && append-ldflags -static + + # build the net-bootable grub first, but only if "netboot" is set + if use netboot ; then + econf \ + --libdir=/lib \ + --datadir=/usr/lib/grub \ + --exec-prefix=/ \ + --disable-auto-linux-mem-opt \ + --enable-diskless \ + --enable-{3c{5{03,07,09,29,95},90x},cs89x0,davicom,depca,eepro{,100}} \ + --enable-{epic100,exos205,ni5210,lance,ne2100,ni{50,65}10,natsemi} \ + --enable-{ne,ns8390,wd,otulip,rtl8139,sis900,sk-g16,smc9000,tiara} \ + --enable-{tulip,via-rhine,w89c840} || die "netboot econf failed" + + emake w89c840_o_CFLAGS="-O" || die "making netboot stuff" + + mv -f stage2/{nbgrub,pxegrub} "${S}"/ + mv -f stage2/stage2 stage2/stage2.netboot + + make clean || die "make clean failed" + fi + + # Now build the regular grub + # Note that FFS and UFS2 support are broken for now - stage1_5 files too big + econf \ + --libdir=/lib \ + --datadir=/usr/lib/grub \ + --exec-prefix=/ \ + --disable-auto-linux-mem-opt || die "econf failed" + emake || die "making regular stuff" +} + +src_test() { + # non-default block size also give false pass/fails. + unset BLOCK_SIZE + make check || die "make check failed" +} + +src_install() { + make DESTDIR="${D}" install || die + if use netboot ; then + exeinto /usr/lib/grub/${CHOST} + doexe nbgrub pxegrub stage2/stage2.netboot || die "netboot install" + fi + + insinto /boot/grub + doins "${DISTDIR}"/splash.xpm.gz + newins docs/menu.lst grub.conf.sample + + dodoc AUTHORS BUGS ChangeLog NEWS README THANKS TODO + newdoc docs/menu.lst grub.conf.sample +} + +setup_boot_dir() { + local dir="${1}" + + [[ ! -e ${dir} ]] && die "${dir} does not exist!" + [[ ! -e ${dir}/grub ]] && mkdir "${dir}/grub" + + # change menu.lst to grub.conf + if [[ ! -e ${dir}/grub/grub.conf ]] && [[ -e ${dir}/grub/menu.lst ]] ; then + mv -f "${dir}"/grub/menu.lst "${dir}"/grub/grub.conf + ewarn + ewarn "*** IMPORTANT NOTE: menu.lst has been renamed to grub.conf" + ewarn + fi + + if [[ ! -e ${dir}/grub/menu.lst ]]; then + einfo "Linking from new grub.conf name to menu.lst" + ln -snf grub.conf "${dir}"/grub/menu.lst + fi + + [[ -e ${dir}/grub/stage2 ]] && mv "${dir}"/grub/stage2{,.old} + + einfo "Copying files from /lib/grub and /usr/lib/grub to "${dir}"" + for x in /lib*/grub/*/* /usr/lib*/grub/*/* ; do + [[ -f ${x} ]] && cp -p ${x} "${dir}"/grub/ + done + + if [[ -e ${dir}/grub/grub.conf ]] ; then + egrep \ + -v '^[[:space:]]*(#|$|default|fallback|initrd|password|splashimage|timeout|title)' \ + "${dir}"/grub/grub.conf | \ + /sbin/grub --batch \ + --device-map="${dir}"/grub/device.map \ + > /dev/null + fi +} + +pkg_postinst() { + [[ ${ROOT} != "/" ]] && return 0 + setup_boot_dir /boot + einfo "To install grub files to another device (like a usb stick), just run:" + einfo " emerge --config =${PF}" +} + +pkg_config() { + local dir + einfo "Enter the directory where you want to setup grub:" + read dir + setup_boot_dir ${dir} +} Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/ChangeLog =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/ChangeLog (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/ChangeLog (revision 24557) @@ -0,0 +1,1160 @@ +# ChangeLog for sys-kernel/genkernel +# Copyright 2000-2008 Gentoo Foundation; Distributed under the GPL v2 +# $Header: /var/cvsroot/gentoo-x86/sys-kernel/genkernel/ChangeLog,v 1.204 2008/06/06 16:12:20 wolf31o2 Exp $ + +*genkernel-3.4.10-r1 (06 Jun 2008) + + 06 Jun 2008; Chris Gianelloni + -genkernel-3.4.10.ebuild, +genkernel-3.4.10-r1.ebuild, + genkernel-9999.ebuild: + Fixed the copying of our tarballs. Thanks to Andrew Gaffney for pointing + out that I had forgotten to add this code to the genkernel 3.4.10 ebuild. + +*genkernel-3.4.10 (05 Jun 2008) + + 05 Jun 2008; Chris Gianelloni + -genkernel-3.4.10_pre7.ebuild, -genkernel-3.4.10_pre8.ebuild, + -genkernel-3.4.10_pre9.ebuild, -genkernel-3.4.10_pre10.ebuild, + +genkernel-3.4.10.ebuild, genkernel-9999.ebuild: + Version bump to 3.4.10 for the 2008.0 release and cleaning up the + pre-release versions. + +*genkernel-3.4.10_pre10 (25 Apr 2008) + + 25 Apr 2008; Chris Gianelloni + +genkernel-3.4.10_pre10.ebuild: + Version bump to 3.4.10_pre10 for testing. + +*genkernel-3.4.10_pre9 (10 Apr 2008) + + 10 Apr 2008; Chris Gianelloni + +genkernel-3.4.10_pre9.ebuild: + Version bump to 3.4.10_pre9 for testing. + +*genkernel-3.4.10_pre8 (04 Apr 2008) + + 04 Apr 2008; Chris Gianelloni + +genkernel-3.4.10_pre8.ebuild: + Version bump to 3.4.10_pre8 for testing. + + 27 Mar 2008; Chris Gianelloni + -genkernel-3.4.10_pre3.ebuild, -genkernel-3.4.10_pre4.ebuild, + -genkernel-3.4.10_pre5.ebuild, -genkernel-3.4.10_pre6.ebuild: + Removing older versions. + + 14 Mar 2008; Chris Gianelloni genkernel-9999.ebuild: + Sync -9999 ebuild with new _pre7 ebuild and fix KEYWORDS. + +*genkernel-3.4.10_pre7 (14 Mar 2008) +*genkernel-3.4.10_pre6 (14 Mar 2008) + + 14 Mar 2008; Chris Gianelloni + +genkernel-3.4.10_pre6.ebuild, +genkernel-3.4.10_pre7.ebuild: + Added 3.4.10_pre6 and 3.4.10_pre7 for testing. + +*genkernel-3.4.10_pre5 (13 Mar 2008) + + 13 Mar 2008; Chris Gianelloni + +genkernel-3.4.10_pre5.ebuild: + Switch back to ~arch KEYWORDS. + + 12 Mar 2008; Robin H. Johnson genkernel-9999.ebuild: + Lowercase S for sed. + + 12 Mar 2008; Chris Gianelloni genkernel-9999.ebuild: + Updated the live SVN ebuild to work with SVN HEAD, which uses a newer busybox. + +*genkernel-3.4.10_pre4 (04 Mar 2008) + + 04 Mar 2008; Chris Gianelloni + +genkernel-3.4.10_pre4.ebuild: + Added 3.4.10_pre4 for testing. + +*genkernel-3.4.10_pre3 (21 Feb 2008) + + 21 Feb 2008; Chris Gianelloni + -genkernel-3.4.9_pre6.ebuild, -genkernel-3.4.10_pre2.ebuild, + +genkernel-3.4.10_pre3.ebuild, genkernel-9999.ebuild: + Version bump to 3.4.10_pre3 for testing. + + 15 Feb 2008; Chris Gianelloni + -genkernel-3.4.10_pre1.ebuild, genkernel-3.4.10_pre2.ebuild, + genkernel-9999.ebuild: + I've modified the ebuilds to use dobin for installing the main genkernel + script and to install AUTHORS via dodoc. Removing the 3.4.10_pre1 ebuild, + since I don't really want anyone using that version. + +*genkernel-3.4.10_pre2 (14 Feb 2008) + + 14 Feb 2008; Chris Gianelloni + +genkernel-3.4.10_pre2.ebuild, genkernel-9999.ebuild: + Version bump to 3.4.10_pre2 for testing and bumping genkernel-pkg revision. + +*genkernel-9999 (14 Feb 2008) +*genkernel-3.4.10_pre1 (14 Feb 2008) + + 14 Feb 2008; Chris Gianelloni + -files/genkernel-3.4.9_pre9-lvm2fix.patch, genkernel-3.4.9.ebuild, + +genkernel-3.4.10_pre1.ebuild, +genkernel-9999.ebuild: + Version bump to 3.4.10_pre1 for testing and adding a 9999 ebuild to pull + directly from subversion, to ease in testing. This closes bug #206039, bug + #206703, bug #207159, bug #207895, bug #208477, bug #208593, and bug + #209182. + + 14 Jan 2008; Chris Gianelloni + genkernel-3.4.9.ebuild: + Stable on all arches. This fixes more bugs than I can recall, versus 3.4.8, + and even versus 3.4.9_pre6, so I hope that everyone enjoys the shiny newness + of it all. + +*genkernel-3.4.9 (12 Jan 2008) + + 12 Jan 2008; Chris Gianelloni + -genkernel-3.4.8.ebuild, -genkernel-3.4.9_pre11.ebuild, + -genkernel-3.4.9_pre12.ebuild, +genkernel-3.4.9.ebuild: + Version bump to 3.4.9 FINAL, which should be fast-tracked to stable. This + closes bug #162962, bug #182818, bug #194752, bug #203301, and bug #204087. + +*genkernel-3.4.9_pre12 (27 Dec 2007) + + 27 Dec 2007; Chris Gianelloni + -genkernel-3.4.9_pre9-r1.ebuild, -genkernel-3.4.9_pre10.ebuild, + +genkernel-3.4.9_pre12.ebuild: + Version bump to 3.4.9_pre12. This will hopefully be going stable soon. Fixes + bug #162962, bug #201159, bug #201442, bug #201482, and bug #202266. + +*genkernel-3.4.9_pre11 (03 Dec 2007) + + 03 Dec 2007; Chris Gianelloni + -genkernel-3.4.9_pre7.ebuild, -genkernel-3.4.9_pre8.ebuild, + -genkernel-3.4.9_pre9.ebuild, +genkernel-3.4.9_pre11.ebuild: + Version bump to 3.4.9_pre11 for testing. + +*genkernel-3.4.9_pre10 (28 Nov 2007) + + 28 Nov 2007; Chris Gianelloni + +genkernel-3.4.9_pre10.ebuild: + Version bump. This closes bug #188273, bug #198892, bug #199701, bug + #200071, and bug #200161. + +*genkernel-3.4.9_pre9-r1 (16 Nov 2007) + + 16 Nov 2007; Chris Gianelloni + +files/genkernel-3.4.9_pre9-lvm2fix.patch, + +genkernel-3.4.9_pre9-r1.ebuild: + Revision bumping to add a patch from Robin Johnson to + fix LVM. Closing bug #198546. + +*genkernel-3.4.9_pre9 (14 Nov 2007) + + 14 Nov 2007; Chris Gianelloni + +genkernel-3.4.9_pre9.ebuild: + Version bump to 3.4.9_pre9 for testing and closing bug #198440 and bug #198694. + +*genkernel-3.4.9_pre8 (07 Nov 2007) + + 07 Nov 2007; Chris Gianelloni + +genkernel-3.4.9_pre8.ebuild: + Version bump due to the introduction of a patch from bug #182616 which is + rather invasive. + +*genkernel-3.4.9_pre7 (07 Nov 2007) + + 07 Nov 2007; Chris Gianelloni + +genkernel-3.4.9_pre7.ebuild: + Version bump to 3.4.9_pre7 and closing bug #193826. + + 07 Nov 2007; Chris Gianelloni + -files/suspend-0.5-Makefile.patch, -genkernel-3.4.9_pre2.ebuild, + -genkernel-3.4.9_pre3.ebuild, -genkernel-3.4.9_pre4.ebuild, + -genkernel-3.4.9_pre5.ebuild, genkernel-3.4.9_pre6.ebuild: + Cleaning up quoting, removing older ebuilds, and marking 3.4.9_pre6 stable. + I normally wouldn't stable a pre-release, but we've been using it in release + building and it resolves too many user bugs to not go stable. + +*genkernel-3.4.9_pre6 (02 Nov 2007) + + 02 Nov 2007; Chris Gianelloni + +genkernel-3.4.9_pre6.ebuild: + Version bump to fix bug #197730. + +*genkernel-3.4.9_pre5 (01 Nov 2007) + + 01 Nov 2007; Chris Gianelloni + +genkernel-3.4.9_pre5.ebuild: + Version bump to fix bug #196087 and bug #197582. + +*genkernel-3.4.9_pre4 (30 Oct 2007) + + 30 Oct 2007; Chris Gianelloni + +genkernel-3.4.9_pre4.ebuild: + Version bump to 3.4.9_pre4 for more testing. + +*genkernel-3.4.9_pre3 (17 Sep 2007) + + 17 Sep 2007; Chris Gianelloni + +genkernel-3.4.9_pre3.ebuild: + Version bump to 3.4.9_pre3 for more testing. + +*genkernel-3.4.9_pre2 (30 Aug 2007) + + 30 Aug 2007; Chris Gianelloni + -genkernel-3.4.9_pre1.ebuild, +genkernel-3.4.9_pre2.ebuild: + Version bump to 3.4.9_pre2 to remove suspend support until we have a better + idea of how to support it. + +*genkernel-3.4.9_pre1 (22 Aug 2007) + + 22 Aug 2007; Chris Gianelloni + +genkernel-3.4.9_pre1.ebuild: + Version bump to a pre-release genkernel version. This should resolve bug + #144703, bug #150697, bug #170753, bug #173766, bug #174188, bug #174294, + bug #176543, bug #179480, bug #179739, bug #180111, bug #180161, bug + #180211, bug #183406, bug #185827, bug #186378, bug #186650, bug #186654, + bug #188273, bug #188398, and bug #188954. + + 22 Aug 2007; Chris Gianelloni + -genkernel-3.4.6.ebuild, -genkernel-3.4.7.ebuild, genkernel-3.4.8.ebuild: + Added a small fix for bug #184007 and removing older versions from the tree. + + 24 Jul 2007; Chris Gianelloni + genkernel-3.4.8.ebuild: + Marking stable on all arches since this is what we used for 2007.0 and I was + supposed to mark it stable along with the release. + + 14 May 2007; Thilo Bangert metadata.xml: + add no-herd + + 12 May 2007; Daniel Drake metadata.xml: + Remove kernel herd from metadata + +*genkernel-3.4.8 (12 Apr 2007) + + 12 Apr 2007; Chris Gianelloni + +genkernel-3.4.8.ebuild: + Added a new genkernel version to the tree for the 2007.0 release. This + closes bug #173622, bug #174130, and bug #174188. + +*genkernel-3.4.7 (05 Apr 2007) + + 05 Apr 2007; Chris Gianelloni + -genkernel-3.4.7_pre5.ebuild, +genkernel-3.4.7.ebuild: + Version bump to 3.4.7 of genkernel. This resolves bug #148253, bug #156009, + bug #168500, bug #168664, bug #168719, bug #169383, bug #171911, bug + #171913, bug #171915, bug #172562, bug #173412. Removing older 3.4.7_pre + versions. + + 14 Mar 2007; Chris Gianelloni + -genkernel-3.4.5-r1.ebuild, -genkernel-3.4.7_pre3.ebuild, + -genkernel-3.4.7_pre4.ebuild: + Removing older unused versions. + +*genkernel-3.4.7_pre5 (14 Mar 2007) + + 14 Mar 2007; Chris Gianelloni + +genkernel-3.4.7_pre5.ebuild: + Version bump to 3.4.7_pre5 for testing for 2007.0's release. + + 10 Mar 2007; Chris Gianelloni + genkernel-3.4.7_pre4.ebuild: + Marking genkernel 3.4.7_pre4 unstable, since it was an accident it went + stable. + +*genkernel-3.4.7_pre4 (09 Mar 2007) + + 09 Mar 2007; Chris Gianelloni + +genkernel-3.4.7_pre4.ebuild: + Version bump to the latest pre-release 3.4.7 for testing. + +*genkernel-3.4.7_pre3 (14 Feb 2007) + + 14 Feb 2007; Chris Gianelloni + +genkernel-3.4.7_pre3.ebuild: + Version bump to the latest pre-release version. This version fixes quite a + few bugs and is needed for release testing, so we're adding it to the tree. + This closes bug #114156, bug #152945, bug #160333, bug #160635, bug #161716, + bug #165494, bug #165758, bug #165980, and bug #166179. + + 02 Feb 2007; Chris Gianelloni + genkernel-3.4.6.ebuild: + Stable for 2007.0. + +*genkernel-3.4.6 (04 Jan 2007) + + 04 Jan 2007; Chris Gianelloni + +files/suspend-0.5-Makefile.patch, -genkernel-3.4.1.ebuild, + -genkernel-3.4.4.ebuild, -genkernel-3.4.5.ebuild, + genkernel-3.4.5-r1.ebuild, +genkernel-3.4.6.ebuild: + Removed older versions. Marked 3.4.5-r1 stable. Added 3.4.6 which fixes the + following bugs: #146466, #146714, #149020, #156410, #156445, #156611, + #156640, #157538, #157777, #158017, #158135, #158776. I am planning on using + 3.4.6 or higher for 2007.0, so it is on the fast track to stable. + +*genkernel-3.4.5-r1 (22 Nov 2006) + + 22 Nov 2006; Aron Griffis +files/genkernel.bash, + +genkernel-3.4.5-r1.ebuild: + Add bash-completion #72845 + +*genkernel-3.4.5 (14 Nov 2006) + + 14 Nov 2006; Chris Gianelloni + +genkernel-3.4.5.ebuild: + Version bump to 3.4.5 to close bug #122672, bug #152441, bug #152881, bug + #153217, bug #153516, and bug #153554. + + 10 Nov 2006; Chris Gianelloni + genkernel-3.4.4.ebuild: + Added notice about LUKS changes and closing bug #154593. + + 09 Nov 2006; Chris Gianelloni + genkernel-3.4.1.ebuild, -genkernel-3.4.2.ebuild, -genkernel-3.4.3.ebuild, + genkernel-3.4.4.ebuild: + Marking stable since it is required for 2.6.18 support. This closes bug + #148498 and bug #153801. Cleaning up older ebuilds. + +*genkernel-3.4.4 (25 Oct 2006) + + 25 Oct 2006; Chris Gianelloni + +genkernel-3.4.4.ebuild: + Version bump. Closes bug #151500 (again), bug #151609, bug #152299, and bug + #152441. + +*genkernel-3.4.3 (23 Oct 2006) + + 23 Oct 2006; Chris Gianelloni + +genkernel-3.4.3.ebuild: + Version bump to 3.4.3 and closing bug #142606, bug #145115, and bug #151500. + + 20 Oct 2006; Chris Gianelloni + files/digest-genkernel-3.4.2, ChangeLog, Manifest: + Changed tarball for genkernel-pkg-3.4.tar.bz2 to resolve bug #151500. + + 13 Oct 2006; Chris Gianelloni -genkernel-3.3.10.ebuild, + -genkernel-3.3.11d.ebuild, -genkernel-3.4.0-r1.ebuild, + -files/genkernel-3.4.0-splash.patch: + Removing older versions. + + 13 Oct 2006; Chris Gianelloni + +genkernel-3.4.2.ebuild: + Version bump to 3.4.2, which solves bug #147951, bug #148498, and bug #148499. + + 13 Oct 2006; Chris Gianelloni metadata.xml: + Changed maintainer to genkernel@gentoo.org since plasmaroo has retired. + +*genkernel-3.4.1 (16 Sep 2006) + + 16 Sep 2006; Tim Yamin +genkernel-3.4.1.ebuild: + Fix bugs #131202, #142775, #143442, #143476, #145248, #145802, #147015, and + #147186. + + 18 Aug 2006; Christian Heim metadata.xml: + Reassigning the package to kernel@g.o, the x86-kernel bugzilla-alias is + burried/dead. + +*genkernel-3.4.0-r1 (01 Aug 2006) + + 01 Aug 2006; Chris Gianelloni + +files/genkernel-3.4.0-splash.patch, -genkernel-3.4.0.ebuild, + +genkernel-3.4.0-r1.ebuild: + revision bump to include a patch for the path to splash_geninitramfs. + Closing bug #142412. + + 01 Aug 2006; Chris Gianelloni + -genkernel-3.3.11.ebuild, -genkernel-3.3.11a.ebuild, + -genkernel-3.4.0_pre2.ebuild, genkernel-3.4.0.ebuild: + This needs to be stable so bad that it hurts. It should now support newer + kernels. I'm also cleaning up some of the older versions. + +*genkernel-3.4.0 (31 Jul 2006) + + 31 Jul 2006; +genkernel-3.4.0.ebuild, + -genkernel-3.4.0_pre4.ebuild: + 3.4.0; no changes from 3.4.0_pre4. + +*genkernel-3.4.0_pre4 (20 Jul 2006) + + 20 Jul 2006; -genkernel-3.4.0_pre3.ebuild, + +genkernel-3.4.0_pre4.ebuild: + Fix bug #141153; patch from Pylon. + +*genkernel-3.4.0_pre3 (18 Jul 2006) + + 18 Jul 2006; -genkernel-3.4.0_pre1.ebuild, + +genkernel-3.4.0_pre3.ebuild: + modules_load updates; also fix bug #139866 (thanks to Martin Parm for the + patch). + +*genkernel-3.4.0_pre2 (30 Jun 2006) + + 30 Jun 2006; +genkernel-3.4.0_pre2.ebuild: + PCMCIA modules_load updates. + + 22 Jun 2006; Chris Gianelloni + genkernel-3.4.0_pre1.ebuild: + I've added a commented line with this stable on all arches. This way we can + simply uncomment the line and redigest when doing release snapshots, making + it easier to test. This does not affect the package's KEYWORDS in the tree. + +*genkernel-3.4.0_pre1 (20 Jun 2006) + + 20 Jun 2006; +genkernel-3.4.0_pre1.ebuild: + Fix bugs #107628, #113426, #122421, #127672, #129887, #129965, #130097, + #131769, #132408, #134843, #135413, #135703. + + 27 Apr 2006; Alec Warner + files/digest-genkernel-3.3.10, Manifest: + Fixing SHA256 digest, pass four + +*genkernel-3.3.11d (14 Apr 2006) + + 14 Apr 2006; -genkernel-3.3.11c.ebuild, + +genkernel-3.3.11d.ebuild: + Fix #121616, #128805, #129887, #129910. + +*genkernel-3.3.11c (13 Apr 2006) + + 13 Apr 2006; Chris Gianelloni + +genkernel-3.3.11c.ebuild: + Added a new revision 3.3.11c which upgrades the unionfs support and has some + general code cleanup. This should close bug #114266, bug #117114, bug + #121334, bug #124251, bug #124936, and bug #125380. + + 13 Apr 2006; Chris Gianelloni + genkernel-3.3.11a.ebuild: + Changed 3.3.11a to use klibc 1.2.1 instead of 1.1.16, which should resolve + bug #121846, bug #122629, bug #126316, and bug #128646. + +*genkernel-3.3.11a (06 Apr 2006) + + 06 Apr 2006; +genkernel-3.3.11a.ebuild: + Fix #128806. + + 14 Mar 2006; Chris Gianelloni + genkernel-3.3.11.ebuild: + Marking stable as this was the version used to build 2006.0 and fixes quite + a few bugs, as well as it should have been marked stable at release time, + and I forgot it. Shame on me. + + 10 Mar 2006; genkernel-3.3.10.ebuild, + genkernel-3.3.11.ebuild: + Fix typo; bug #125298. + + 09 Feb 2006; Chris Gianelloni + genkernel-3.3.10.ebuild, genkernel-3.3.11.ebuild: + Added a note about not using reiser4 with genkernel as it is known to be + broken. + + 08 Feb 2006; Chris Gianelloni + genkernel-3.3.11.ebuild: + Added sed to fix a bug in klibc not being uncompressed. + +*genkernel-3.3.11 (08 Feb 2006) + + 08 Feb 2006; Chris Gianelloni + -genkernel-3.3.11_pre8.ebuild, +genkernel-3.3.11.ebuild: + Version bumped to 3.3.11 final. + +*genkernel-3.3.11_pre8 (02 Feb 2006) + + 02 Feb 2006; Chris Gianelloni + -files/bin-keymaps.tar.gz, -genkernel-3.3.11_pre7.ebuild, + +genkernel-3.3.11_pre8.ebuild: + Version bump and closing bug #120698. + +*genkernel-3.3.11_pre7 (27 Jan 2006) + + 27 Jan 2006; Chris Gianelloni + -genkernel-3.1.0c.ebuild, -genkernel-3.1.1b.ebuild, + -genkernel-3.1.6.ebuild, -genkernel-3.3.11_pre6.ebuild, + +genkernel-3.3.11_pre7.ebuild: + Version bump to 3.3.11_pre7. This closes bug #120526. + +*genkernel-3.3.11_pre6 (26 Jan 2006) + + 26 Jan 2006; Chris Gianelloni + -genkernel-3.3.11_pre5.ebuild, +genkernel-3.3.11_pre6.ebuild: + Version bump to 3.3.11_pre6 to resolve some issues with + mixed-userland/kernel environments. + +*genkernel-3.3.11_pre5 (25 Jan 2006) + + 25 Jan 2006; Chris Gianelloni + -genkernel-3.3.11_pre4.ebuild, +genkernel-3.3.11_pre5.ebuild: + Version bump to 3.3.11_pre5 for testing. This includes a patch for + 2.6/klibc/udev on sparc. + +*genkernel-3.3.11_pre4 (24 Jan 2006) + + 24 Jan 2006; Chris Gianelloni + -genkernel-3.3.11_pre3.ebuild, +genkernel-3.3.11_pre4.ebuild: + Version bumped to 3.3.11_pre4 for testing. This includes a patch for ppc64 + compilation with a 32-bit userland. + + 18 Jan 2006; Chris Gianelloni + -genkernel-3.3.11_pre2.ebuild: + Cleaning up older 3.3.11_pre2 ebuild. + +*genkernel-3.3.11_pre3 (12 Jan 2006) + + 12 Jan 2006; Chris Gianelloni + +genkernel-3.3.11_pre3.ebuild: + Added patch to resolve compile issues. + +*genkernel-3.3.11_pre2 (12 Jan 2006) + + 12 Jan 2006; Chris Gianelloni + -genkernel-3.3.11_pre1.ebuild, +genkernel-3.3.11_pre2.ebuild: + Upgraded to 3.3.11_pre2 to fix a problem with busybox and 2.6.15 headers. + +*genkernel-3.3.11_pre1 (11 Jan 2006) + + 11 Jan 2006; Chris Gianelloni + +genkernel-3.3.11_pre1.ebuild: + Added 3.3.11_pre1 for testing klibc-1.1.16 upgrade. + + 06 Jan 2006; Chris Gianelloni + -genkernel-3.3.6.ebuild, genkernel-3.3.10.ebuild: + Marking 3.3.10 stable and removing older 3.3 versions. + +*genkernel-3.3.10 (18 Dec 2005) + + 18 Dec 2005; +genkernel-3.3.10.ebuild, + -genkernel-3.3.9.ebuild: + Add a compile_klibc(...) fix for old GNU tars and patch the klibc Makefile to + leave /lib alone. + +*genkernel-3.3.9 (16 Dec 2005) + + 16 Dec 2005; Chris Gianelloni + -genkernel-3.3.8.ebuild, +genkernel-3.3.9.ebuild: + Version bumped to 3.3.9 and closing bugs: #100637, #109196, #113634, + #114496, #113684. + + 23 Nov 2005; Eric Edgar genkernel-3.3.8.ebuild: + Update 3.3.8 ebuild to point to the proper dmraid binary + +*genkernel-3.3.8 (23 Nov 2005) + + 23 Nov 2005; Eric Edgar -genkernel-3.3.7.ebuild, + +genkernel-3.3.8.ebuild: + Fix bug 113287;Version bump 3.3.8 + +*genkernel-3.3.7 (22 Nov 2005) + + 22 Nov 2005; Eric Edgar +genkernel-3.3.7.ebuild: + Version bump. Fixes bugs: #108371 #109819 #111546 #102006 #103332 #103717 + #103936 #105572 #94762 + +*genkernel-3.3.6 (08 Sep 2005) + + 08 Sep 2005; Chris Gianelloni + -genkernel-3.3.5.ebuild, +genkernel-3.3.6.ebuild: + Version bump. Fixes bugs: #83276 #88080 #101535 #101599 #101716 #102006 + #102407 #102491 #102739 + +*genkernel-3.3.5 (30 Jul 2005) + + 30 Jul 2005; -genkernel-3.3.4.ebuild, + +genkernel-3.3.5.ebuild: + Fix initrd cosmetics and PPC compiles without --genzimage. + +*genkernel-3.3.4 (30 Jul 2005) + + 30 Jul 2005; -genkernel-3.3.3.ebuild, + +genkernel-3.3.4.ebuild: + Fix LiveCD support for BladeCenter. + +*genkernel-3.3.3 (29 Jul 2005) + + 29 Jul 2005; Chris Gianelloni + -genkernel-3.3.2.ebuild, +genkernel-3.3.3.ebuild: + Version bump. Fixes compiling on sparc. + +*genkernel-3.3.2 (28 Jul 2005) + + 28 Jul 2005; -genkernel-3.3.1.ebuild, + +genkernel-3.3.2.ebuild: + Fix PPC logic when used without --genzimage. + +*genkernel-3.3.1 (28 Jul 2005) + + 28 Jul 2005; +genkernel-3.3.1.ebuild, + -genkernel-3.3.ebuild: + Fix #100144, #100169, #100583. + +*genkernel-3.3 (24 Jul 2005) + + 24 Jul 2005; -genkernel-3.2.10.ebuild, + +genkernel-3.3.ebuild: + Fix #80617, #99636. + +*genkernel-3.2.10 (21 Jul 2005) + + 21 Jul 2005; +genkernel-3.2.10.ebuild, + -genkernel-3.2.9.ebuild: + Fix --minkernpackage issues with Catalyst. + +*genkernel-3.2.9 (19 Jul 2005) + + 19 Jul 2005; Chris Gianelloni + -genkernel-3.2.5.ebuild, -genkernel-3.2.8.ebuild, +genkernel-3.2.9.ebuild: + Version bumped and cleaning up older versions. + +*genkernel-3.2.8 (19 Jul 2005) + + 19 Jul 2005; -genkernel-3.2.7.ebuild, + +genkernel-3.2.8.ebuild: + Add --postconf support. + +*genkernel-3.2.7 (16 Jul 2005) + + 16 Jul 2005; +genkernel-3.2.7.ebuild: + Fix #98886. + +*genkernel-3.2.5 (14 Jul 2005) + + 14 Jul 2005; -genkernel-3.2.4.ebuild, + +genkernel-3.2.5.ebuild: + Split out pkg/. + +*genkernel-3.2.4 (14 Jul 2005) + + 14 Jul 2005; -genkernel-3.2.3.ebuild, + +genkernel-3.2.4.ebuild: + Fix #80617, #98944, add GRUB :root_device support and add a more versatile + grub.conf updater. + +*genkernel-3.2.3 (14 Jul 2005) + + 14 Jul 2005; -genkernel-3.2.2.ebuild, + +genkernel-3.2.3.ebuild: + Fix #97672, #98886, #98893, #98897; fix real_root=/dev/mdX. + +*genkernel-3.2.2 (12 Jul 2005) + + 12 Jul 2005; -genkernel-3.2.1.ebuild, + +genkernel-3.2.2.ebuild: + Fix #83771, #97700, #98590, #98594, #98661, #98746. + +*genkernel-3.2.1 (10 Jul 2005) + + 10 Jul 2005; -genkernel-3.2.0.ebuild, + +genkernel-3.2.1.ebuild: + Fix #93178, #98436, #98501. + +*genkernel-3.2.0 (09 Jul 2005) + + 09 Jul 2005; Chris Gianelloni + -genkernel-3.2.0_pre18.ebuild, +genkernel-3.2.0.ebuild: + Genkernel 3.2.0... ph34r... + +*genkernel-3.2.0_pre18 (03 Jul 2005) + + 03 Jul 2005; -genkernel-3.2.0_pre17.ebuild, + +genkernel-3.2.0_pre18.ebuild: + Clean up /dev issues near init with the linuxrc. + +*genkernel-3.2.0_pre17 (29 Jun 2005) + + 29 Jun 2005; -genkernel-3.2.0_pre15.ebuild, + +genkernel-3.2.0_pre17.ebuild: + Revert udev back to 054; update unionfs code. + + 28 Jun 2005; -genkernel-1.6.ebuild, + -genkernel-1.8.ebuild, -genkernel-1.9.ebuild, -genkernel-3.0.1.ebuild, + -genkernel-3.0.2.ebuild, -genkernel-3.1.5.ebuild: + Remove stale versions. + +*genkernel-3.2.0_pre15 (28 Jun 2005) + + 28 Jun 2005; -genkernel-3.2.0_pre14.ebuild, + +genkernel-3.2.0_pre15.ebuild: + Fix #97281. + +*genkernel-3.2.0_pre14 (27 Jun 2005) + + 27 Jun 2005; -genkernel-3.2.0_pre12.ebuild, + +genkernel-3.2.0_pre14.ebuild: + Update udev with patch for cross-compile on SPARC64. + +*genkernel-3.2.0_pre12 (24 Jun 2005) + + 24 Jun 2005; -genkernel-3.2.0_pre11.ebuild, + +genkernel-3.2.0_pre12.ebuild: + Syntax and modules_load updates. + +*genkernel-3.2.0_pre11 (23 Jun 2005) + + 23 Jun 2005; -genkernel-3.2.0_pre10.ebuild, + +genkernel-3.2.0_pre11.ebuild: + Fix gen_compile.sh. + +*genkernel-3.2.0_pre10 (22 Jun 2005) + + 22 Jun 2005; +genkernel-3.2.0_pre10.ebuild, + -genkernel-3.2.0_pre9.ebuild: + Fix #76082, #86487, #87673, #95993. + +*genkernel-3.2.0_pre9 (20 Jun 2005) + + 20 Jun 2005; -genkernel-3.2.0_pre8.ebuild, + +genkernel-3.2.0_pre9.ebuild: + Fix catalyst invocation issues. + +*genkernel-3.2.0_pre8 (17 Jun 2005) + + 17 Jun 2005; Chris Gianelloni + -files/genkernel-3.2.0_pre7-arch.diff, -genkernel-3.2.0_pre7-r1.ebuild, + +genkernel-3.2.0_pre8.ebuild: + Version bumped to 3.2.0_pre8 to fix a bug with localversion and 2.4 kernels. + + 17 Jun 2005; Chris Gianelloni + files/genkernel-3.2.0_pre7-arch.diff, genkernel-3.2.0_pre7-r1.ebuild: + Fixed patch and added inherit eutils for epatch support. + +*genkernel-3.2.0_pre7-r1 (16 Jun 2005) + + 16 Jun 2005; Chris Gianelloni + +files/genkernel-3.2.0_pre7-arch.diff, -genkernel-3.2.0_pre7.ebuild, + +genkernel-3.2.0_pre7-r1.ebuild: + Added arch patch. + +*genkernel-3.2.0_pre7 (16 Jun 2005) + + 16 Jun 2005; -genkernel-3.2.0_pre5.ebuild, + +genkernel-3.2.0_pre7.ebuild: + Fix LOCALVERSION (#88987) and add sleep timeout for loop module detection; + thanks rocket. + +*genkernel-3.2.0_pre5 (15 Jun 2005) + + 15 Jun 2005; -genkernel-3.2.0_pre4.ebuild, + +genkernel-3.2.0_pre5.ebuild: + Fix star compatibility, add updated Pegasos config and initramfs support for + Pegasos kernels. + +*genkernel-3.2.0_pre4 (14 Jun 2005) + + 14 Jun 2005; -genkernel-3.2.0_pre3.ebuild, + +genkernel-3.2.0_pre4.ebuild: + Add rocket's unionfs patch and linuxrc cleanup; fix Pegasos with regard to + initramfs. + +*genkernel-3.2.0_pre3 (07 Jun 2005) + + 07 Jun 2005; -genkernel-3.2.0_pre2.ebuild, + +genkernel-3.2.0_pre3.ebuild: + Fix minkernpackage bug. + +*genkernel-3.2.0_pre2 (03 Jun 2005) + + 03 Jun 2005; +genkernel-3.2.0_pre2.ebuild: + Fix #73158, #77377, #88682, #88988, #94680. Masked until we know this all + works right. + +*genkernel-2.1 (11 May 2005) + + 11 May 2005; -genkernel-2.0.ebuild, + -genkernel-2.1.ebuild: + Remove old unavailable versions. + + 30 Apr 2005; genkernel-3.1.6.ebuild: + Fix #80156. + +*genkernel-3.1.6 (01 Apr 2005) + + 01 Apr 2005; +genkernel-3.1.6.ebuild: + Fix #87117. + +*genkernel-3.1.5 (30 Mar 2005) + + 30 Mar 2005; +genkernel-3.1.5.ebuild: + Sync Portage to latest genkernel version. + +*genkernel-3.1.1b (02 Mar 2005) + + 02 Mar 2005; -genkernel-3.1.1a.ebuild, + +genkernel-3.1.1b.ebuild: + Upgrade dmraid to 1.0.0.rc6. + +*genkernel-3.1.1a (01 Mar 2005) + + 01 Mar 2005; -genkernel-3.1.1.ebuild, + +genkernel-3.1.1a.ebuild: + Fix #83672. + + 28 Feb 2005; genkernel-3.1.1.ebuild: + Add device-mapper to the LiveCD fetch-list. + +*genkernel-3.1.1 (28 Feb 2005) + + 28 Feb 2005; -genkernel-3.1.0l.ebuild, + +genkernel-3.1.1.ebuild: + Fix 2.4 module compilation, add livecd USE support for fetching LVM2 and + DMRAID source into /usr/share/genkernel/pkg. + +*genkernel-3.1.0l (27 Feb 2005) + + 27 Feb 2005; -genkernel-3.1.0k.ebuild, + +genkernel-3.1.0l.ebuild: + Add MODULES_SATA support; make udev static on non-x86 architectures. + + 25 Feb 2005; genkernel-3.1.0k.ebuild: + Mention available documentation, #77309. + +*genkernel-3.1.0k (23 Feb 2005) + + 23 Feb 2005; -genkernel-3.1.0j.ebuild, + +genkernel-3.1.0k.ebuild: + Fix #83048. + +*genkernel-3.1.0j (18 Feb 2005) + + 18 Feb 2005; -genkernel-3.1.0h.ebuild, + +genkernel-3.1.0j.ebuild: + Fix #58686 and #80716. + +*genkernel-3.1.0h (30 Jan 2005) + + 30 Jan 2005; -genkernel-3.1.0g.ebuild, + +genkernel-3.1.0h.ebuild: + Fix bug #79999. + + 29 Jan 2005; Chris Gianelloni + genkernel-3.1.0c.ebuild: + Marking 3.1.0c stable on ppc64 since there were no stable versions. + +*genkernel-3.1.0g (27 Jan 2005) + + 27 Jan 2005; -genkernel-3.0.2a.ebuild, + -genkernel-3.0.2b.ebuild, -genkernel-3.0.2c-r1.ebuild, + -genkernel-3.0.2c.ebuild, -genkernel-3.0.2d.ebuild, + -genkernel-3.0.2g.ebuild, -genkernel-3.1.0a.ebuild, + -genkernel-3.1.0b.ebuild, -genkernel-3.1.0f.ebuild, + +genkernel-3.1.0g.ebuild: + Version bump; fix bug #79755. Remove ebuilds for old versions. + +*genkernel-3.1.0f (26 Jan 2005) + + 26 Jan 2005; -genkernel-3.1.0e.ebuild, + +genkernel-3.1.0f.ebuild: + Version bump; bug #79502. + +*genkernel-3.1.0e (22 Jan 2005) + + 22 Jan 2005; -genkernel-3.1.0d.ebuild, + +genkernel-3.1.0e.ebuild: + Version bump. Closes bugs #73356, #74758, #77277, #77363, #78636. + +*genkernel-3.1.0d (21 Dec 2004) + + 21 Dec 2004; +genkernel-3.1.0d.ebuild: + Version bump. Closes bugs #73112, #70193 and #73054. + +*genkernel-3.1.0c (30 Nov 2004) + + 30 Nov 2004; +genkernel-3.1.0c.ebuild: + Version bump. Closes bugs #61827, #64864, #66198, #67246, #69745, #70193, + #72129, #72253, #72342, #72510. + +*genkernel-3.1.0b (23 Oct 2004) + + 23 Oct 2004; -genkernel-3.1.0.ebuild, + +genkernel-3.1.0a.ebuild, +genkernel-3.1.0b.ebuild: + Version bump - fixes udev keymap issues and bug #68518. + + 22 Oct 2004; Chris Gianelloni + genkernel-3.1.0.ebuild: + Added missing ppc64 KEYWORD. + +*genkernel-3.1.0 (21 Oct 2004) + + 21 Oct 2004; Chris Gianelloni + +genkernel-3.1.0.ebuild: + Finally adding 3.1.0 to cvs. Blame plasmaroo. + +*genkernel-3.0.2c-r1 (02 Oct 2004) + + 02 Oct 2004; Travis Tilley +genkernel-3.0.2c-r1.ebuild: + updated the x86_64 kernel configs for the latest amd64 stable genkernel + +*genkernel-3.0.2g (19 Sep 2004) + + 19 Sep 2004; -genkernel-3.0.2f.ebuild, + +genkernel-3.0.2g.ebuild: + Version bump - closes bugs #57576, #58360, #60862 and #62365. + +*genkernel-3.1.0 (21 Oct 2004) + + 21 Oct 2004; Chris Gianelloni + +genkernel-3.1.0.ebuild: + Finally adding 3.1.0 to cvs. Blame plasmaroo. + +*genkernel-3.0.2f (23 Aug 2004) + + 23 Aug 2004; -genkernel-3.0.2e.ebuild, + +genkernel-3.0.2f.ebuild: + Version bump - closes bugs #57867, #58663, #58816, #59310 and #59687. + + 28 Jul 2004; genkernel-3.0.2c.ebuild, + genkernel-3.0.2d.ebuild, genkernel-3.0.2e.ebuild: + Added a local bootsplash USE flag; bug #37015. + +*genkernel-3.0.2e (28 Jul 2004) + + 28 Jul 2004; +genkernel-3.0.2e.ebuild: + Version bump - fixes bugs #47666, #49328, #52558, #52561, #57297, + #57576, #57748, #57749, #57751, #57752, #57761, #57836, #57865, + #57953. Thanks to Martin Parm, Mathias Gug and Bob Barry who contributed + patches! + + 18 Jul 2004; Tom Gall genkernel-3.0.2d.ebuild: + stable on ppc64, bug #57487 + +*genkernel-3.0.2d (15 Jul 2004) + + 15 Jul 2004; +genkernel-3.0.2d.ebuild: + Added a modutils fix for HPPA. + +*genkernel-3.0.2c (12 Jul 2004) + + 12 Jul 2004; +genkernel-3.0.2c.ebuild: + Version bump - closes bug #54455, #55376 and adds more PCMCIA modules to the + x86 module list. + + 25 Jun 2004; Chris Gianelloni genkernel-3.0.2b.ebuild: + Marking stable on amd64 x86 sparc hppa alpha ppc, as requested by plasmaroo. + + 15 Jun 2004; John Davis genkernel-3.0.2b.ebuild: + keywording for arm and s390 (catalyst dependency) + +*genkernel-3.0.2b (12 Jun 2004) + + 12 Jun 2004; +genkernel-3.0.2b.ebuild: + Version bump. Fixes bugs #46167, #46278, #48219, #48308, #49728, #51395, + and #51948. + + 31 May 2004; Pieter Van den Abeele + genkernel-1.6.ebuild, genkernel-1.8.ebuild, genkernel-1.9.ebuild, + genkernel-3.0.1.ebuild: + Masked genkernel-3.0.1.ebuild stable for ppc + + 31 May 2004; Pieter Van den Abeele + genkernel-1.6.ebuild, genkernel-1.8.ebuild, genkernel-1.9.ebuild: + Masked genkernel-1.9.ebuild broken for ppc + + 31 May 2004; Pieter Van den Abeele + genkernel-1.6.ebuild, genkernel-1.8.ebuild: + Masked genkernel-1.8.ebuild broken for ppc + + 31 May 2004; Pieter Van den Abeele + genkernel-1.6.ebuild: + Masked genkernel-1.6.ebuild broken for ppc + +*genkernel-3.0.2a (11 Apr 2004) + + 11 Apr 2004; genkernel-3.0.2a.ebuild: + Version bump; closes bugs #46641 and #46941. + +*genkernel-3.0.2 (30 Mar 2004) + + 30 Mar 2004; genkernel-3.0.2.ebuild: + Version bump - GenKernel 3.0.2 is released as version 3.0.2_rc2 with support + for the "docache" boot parameter for LiveCDs. + +*genkernel-3.0.2_rc2 (27 Mar 2004) + + 27 Mar 2004; Tim Yamin genkernel-3.0.2_rc2.ebuild, + Version bump. Closes bug #45731 and updates the SPARC configuration scripts. + + 21 Mar 2004; Tim Yamin + genkernel-3.0.2_rc1.ebuild: + Version bump. Closes bugs #34948, #37371, #41129, #41166, + #42725, #42815, #44127, #44556, #44601. + + 12 Mar 2004; Seemant Kulleen + genkernel-3.0.1_rc2.ebuild: + More cleanups and point SRC_URI to mirrors. + + 09 Mar 2004; Seemant Kulleen genkernel-3.0.1.ebuild, + genkernel-3.0.1_beta10.ebuild, genkernel-3.0.1_beta11.ebuild, + genkernel-3.0.1_beta12.ebuild, genkernel-3.0.1_rc1.ebuild, + genkernel-3.0.ebuild: + Removed crufty versions, adjusted SRC_URI to gentoo mirrors, and fixed some + syntax. + + 01 Mar 2004; Brad House genkernel-3.0.1.ebuild: + Break digests for genkernel, bad version was uploaded. + +*genkernel-3.0.1 (27 Feb 2004) + + 27 Feb 2004; Daniel Robbins + New release with bootsplash fixes + (now uses the "default" bootsplash symlink.) + +*genkernel-3.0.1_rc2 (25 Feb 2004) + + 25 Feb 2004; Brad House genkernel-3.0.1_rc2.ebuild: + Version bump, livecd error message fix, alpha fix, reiserfs now static in 2.4 + x86 kernel. + +*genkernel-3.0.1_rc1 (22 Feb 2004) + + 22 Feb 2004; Brad House genkernel-3.0.1_rc1.ebuild: + Gcloop fixes, and amd64 kernel config updates. + + 14 Feb 2004; Brad House + genkernel-3.0.1_beta12.ebuild: + Added the ppc keyword. + +*genkernel-3.0.1_beta12 (14 Feb 2004) + + 14 Feb 2004; Brad House genkernel-3.0.1_beta1.ebuild, + genkernel-3.0.1_beta12.ebuild, genkernel-3.0.1_beta2.ebuild, + genkernel-3.0.1_beta4.ebuild, genkernel-3.0.1_beta5.ebuild, + genkernel-3.0.1_beta6.ebuild, genkernel-3.0.1_beta7.ebuild, + genkernel-3.0.1_beta8.ebuild, genkernel-3.0.1_beta9.ebuild: + bugfixes for initrd and console remapping/initrd unmounting + ppc profile added + default TEMP location moved to /var/tmp/genkernel + squashfs and gcloop support added + +*genkernel-3.0.1_beta11 (08 Feb 2004) + + 08 Feb 2004; Brad House + genkernel-3.0.1_beta11.ebuild: + fix for typo + +*genkernel-3.0.1_beta10 (07 Feb 2004) + + 07 Feb 2004; Brad House + genkernel-3.0.1_beta10.ebuild: + version bump for fixes + +*genkernel-3.0.1_beta9 (29 Jan 2004) + + 29 Jan 2004; Brad House genkernel-3.0.1_beta9.ebuild: + fixes as per bugs on bugs.gentoo.org + +*genkernel-3.0.1_beta8 (27 Jan 2004) + + 27 Jan 2004; Brad House genkernel-3.0.1_beta8.ebuild: + config file handling as per many bug reports, also updated x86 2.4 and 2.6 + kernel configs + + 24 Jan 2004; Brad House : + bump + + 12 Jan 2004; Brad House genkernel-3.0.1_beta4.ebuild: + ok, people need to be using genkernel3 now on x86 + +*genkernel-3.0.1_beta4 (11 Jan 2004) + + 11 Jan 2004; Brad House genkernel-3.0.1_beta4.ebuild: + parisc and livecd fixes + +*genkernel-3.0.1_beta3 (07 Jan 2004) + + 11 Jan 2004; Guy Martin genkernel-3.0.1_beta3.ebuild: + Marked stable on hppa. + + 07 Jan 2004; Brad House genkernel-3.0.1_beta3.ebuild: + bump version, bugfixes + +*genkernel-3.0.1_beta2 (24 Dec 2003) + + 24 Dec 2003; Brad House genkernel-3.0.1_beta2.ebuild: + fix nobootsplash + +*genkernel-3.0.1_beta1 (24 Dec 2003) + + 24 Dec 2003; Brad House genkernel-3.0.1_beta1.ebuild: + bump version to fix bugs + +*genkernel-3.0 (22 Dec 2003) + + 22 Dec 2003; Brad House genkernel-3.0.ebuild: + Experiemental genkernel 3.0 + + 28 Oct 2003; Bob Johnson : + Added progressbar, bumped busybox version, started the new 2.6 initrd... + + 24 Oct 2003; Brad House genkernel-1.9.ebuild: + Added '~amd64' flag. + + 19 Oct 2003; zhen genkernel-2.0.ebuild: + OK, we are building fine now. I have left it masked until I can verify that it + can actually build 2.6 kernels ok. + + 19 Oct 2003; zhen genkernel-2.0.ebuild: + This should have been masked. + + 19 Oct 2003; Heinrich Wendel genkernel-2.0.ebuild: + Bug #31533. + + 19 Oct 2003; zhen genkernel-2.0.ebuild: + Updating paths for file locations, this should fix some problems. + Also marking -* because no one should emerge it until it is complete. + +*genkernel-2.0 (19 Oct 2003) + + 19 Oct 2003; zhen genkernel-2.0.ebuild: + 2.0 is the testing only ebuild for 2.6 support. Adding into the tree. + + 18 Oct 2003; zhen files/bin-keymaps.tar.gz: + Moving some source around - bin-keymaps will be added in here instead + of being included in the genkernel tarball. + +*genkernel-1.9 (07 Oct 2003) + + 10 Oct 2003; Bob Johnson + Added MRPROPER setting in /etc/kernel/settings (yes/no) + --myconfig will use /etc/kernels/myconfig. + --buildpkg will not write anything to local system now. + INITRD_SIZE fixed in settings + --livecd will build LiveCD kernel/initrd. + --help now works. + +*genkernel-1.8 (11 Sep 2003) + + 11 Sep 2003; Daniel Robbins : important bug fix; in 1.7, + Modules were not getting copied to the initrd. Now fixed. + +*genkernel-1.7 (10 Sep 2003) + + 10 Sep 2003; Daniel Robbins : Genkernel now FHS-compliant; + Sed redirect fix. Also, MAKEOPTS now comes from Portage/make.conf. + This closes bugs #27920, #27948. + +*genkernel-1.6 (03 Sep 2003) + + 03 Sep 2003; Daniel Robbins : Fixed various bugs: + 26336, 26346, 26822, 26901, 27446, 27871 (some duplicates)... + +*genkernel-1.5 (31 Jul 2003) + + 31 Jul 2003; Daniel Robbins : New version with + support for System.map. + +*genkernel-1.4 (30 Jul 2003) + + 30 Jul 2003; Daniel Robbins : New version with + lots of fixes. + +*genkernel-1.2 (14 07 2003) + + 14 07 2003; Bob Johnson + Minor typo fixes. + +*genkernel-1.1 (26 04 2003) + + 26 04 2003; Bob Johnson + + - Added kernel package option (--buildpkg). + - Make fail on missing $KV. + +*genkernel-1.0 (14 04 2003) + + 14 04 2003; Bob Johnson + Initial import. Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/Manifest (revision 24557) @@ -0,0 +1,13 @@ +AUX genkernel-add-arcmsr.patch 1430 RMD160 fbd8d5b8cca661864dad5a98d83b315082f65ab3 SHA1 20d9b230d2697a302cea382a86d0407b72902732 SHA256 c452856324bee77d9b0a40784017bf831cc182152124c8e1e45b696f298a4e3b +AUX genkernel.bash 2438 RMD160 b3d8e4436d8768c07d18c5aaff9c14fc16641729 SHA1 e15a9c6ef506aa6872a6a97d26751246dedded04 SHA256 e7de9771ba79ef878e2c9a21a1c1720f092c6047d3cc7d4a49ff6b90e7e2c248 +AUX suspend-0.5-Makefile.patch 2072 RMD160 851d6ba7fa13c925e9551f982fed12ba6c5daa75 SHA1 5fcc15537de62ea1ca97376acff73a71407a0033 SHA256 9faba4c78113e8c44ce792be3b6c41d4a8dbf9c2b33b8665ceab906024e5a492 +DIST LVM2.2.02.28.tgz 531905 RMD160 bf6f25db6e34bf362353ae9557691a7f01962502 SHA1 6eb2f6e1228b017ad23d0b7f2bb12fd113f96616 SHA256 2d8d7b123e427d0bfb7471d7c17dfd88890eb98257ca22ea3be5d7307e83aceb +DIST busybox-1.7.4.tar.bz2 1727940 RMD160 a4b71a246449f88a59acda6ecf019f66b9b7c19b SHA1 e5dcaa25525ca63663c3d4e7027897fed54a4f95 SHA256 8480a7c1b9ac0fddff55e8daae807ccf0a47aa292137fa8c9f97cf0733b459c7 +DIST device-mapper.1.02.22.tgz 19981 RMD160 70c8d0157a0856d81f7240b4123ac61c747175dd SHA1 e29c250e1b052ce94dfc26c4ded02b6cb19532c0 SHA256 eccd511ffb41eea917bb0048f3c29ffb57570f6da2f97a60bafb45770c972e6d +DIST dmraid-1.0.0.rc14.tar.bz2 164234 RMD160 9cd238a981cfef9c5c1f2f1d6466b70c95ec9c7c SHA1 2b3284db46a995967d88993ae5ae36b57c513bc4 SHA256 a777354d6d69a9b58d84966cc7b37bc3f5c89539f885ad25fd874ed1c388fbec +DIST e2fsprogs-1.39.tar.gz 3695111 RMD160 67e77a7cbf0e719fa8b5a8ac6285b334e1c3481b SHA1 c2dad0c9bd76701eafe3804e2a7d072aa198f046 SHA256 dc2033447e69a1612881151f6f163e8a3b80e51d16f5e8b3576f3f8ccb325ea3 +DIST genkernel-3.4.10.tar.bz2 210822 RMD160 19327712c7140336db08a466a19f164df11eac7a SHA1 a672b1b81c6b607533f2576f4bf3adaf620f9f54 SHA256 43e7ae3495e5f8e4ead919b059444bfa393bf83d36c14390206892f82ad37685 +EBUILD genkernel-3.4.10-r1.ebuild 4279 RMD160 a5f90167fc6ade3a3cb6be6ce65d1a2e8968e795 SHA1 396e3106be9a49c1290dda4476ea8ff8da8202e6 SHA256 b78ada42cf98eea8db55ca261b8cf4c93710b594fe592e9081711c16dac3cac5 +MISC .genkernel-3.4.6.ebuild.swp 12288 RMD160 f41038382d367adc56162d96d916c39094a59692 SHA1 9aaa77b5d2eddfe537f4cddc3bb66f20b2abda81 SHA256 80d3d78af5040504f5e460dbca1c217a106418ecc3fdeca445caad43ea3d286d +MISC ChangeLog 39647 RMD160 a8ad6c839ef778ee647194b70ca514e4410b1f93 SHA1 79a268b08aa165ece76874225e32332e97544726 SHA256 85aa458edf94b93d7e4967ed4d1fdbe5981a439591e377e55e70188ffca67ceb +MISC metadata.xml 226 RMD160 8c8b08effbb307ff307346f319d8e7ec6baddd6e SHA1 33172dd797d80d11d35a2b1855e54cd2a737548d SHA256 b7b060498e250d9de733cf433899caf8e8607fac9fa648f84f5c0512a89d794b Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/digest-genkernel-3.4.6 =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/digest-genkernel-3.4.6 (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/digest-genkernel-3.4.6 (revision 24557) @@ -0,0 +1,24 @@ +MD5 87d8615a655b9a6c1a0e234eab2485bf LVM2.2.02.05.tgz 473621 +RMD160 7295f14e64dc25035d46e3512db6cae1b2be5f9b LVM2.2.02.05.tgz 473621 +SHA256 707c596c7a2f11cc08d3b1099995be2a4a3859a5454ed5af1a6c5361a821a402 LVM2.2.02.05.tgz 473621 +MD5 460cc211b03af4048ec90c0de2ecd8f7 device-mapper.1.02.07.tgz 923923 +RMD160 517dd7bde6935235f8c22149ef6e0cb80e6c925b device-mapper.1.02.07.tgz 923923 +SHA256 1e9c16e8f7bcf87bec03103dda63c0d11cdc739f432488b026b4614fd3cabf40 device-mapper.1.02.07.tgz 923923 +MD5 a2a8948de7717e2e76a1291fc1daf9d1 dmraid-1.0.0.rc13.tar.bz2 170497 +RMD160 9b95d68c2177c3bb7c74227c375780d3b4936ef7 dmraid-1.0.0.rc13.tar.bz2 170497 +SHA256 8497196528d50203053de0f733917b1d85aae416d2136c695a07aee5632b12b9 dmraid-1.0.0.rc13.tar.bz2 170497 +MD5 d774d4412bfb80d12cf3a4fdfd59de5a e2fsprogs-1.38.tar.gz 3621193 +RMD160 492071c29e9a0adc1bed0762e795efb6b29a692c e2fsprogs-1.38.tar.gz 3621193 +SHA256 c4e482687d0cff240d02a70fcf423cc14296b6a7869cd8dd42d5404d098e0bb7 e2fsprogs-1.38.tar.gz 3621193 +MD5 e13d5b1d732ddc8541cea0f84f82e265 genkernel-3.4.6.tar.bz2 169270 +RMD160 5d38aeebaab78e55cb49a8dbd63504fae5b8d5be genkernel-3.4.6.tar.bz2 169270 +SHA256 e1d56d8d16e67181777281d479ed2d323586cc971226f2c7d716a409ac5da0bf genkernel-3.4.6.tar.bz2 169270 +MD5 2fa8384b099d53f878a1f8ef49214493 genkernel-pkg-3.4.tar.bz2 1848904 +RMD160 09e1baed7e3703b2f6c304656a81a12ba4594083 genkernel-pkg-3.4.tar.bz2 1848904 +SHA256 2345a16a3b377e3ffde2f0cca7e82cf3566e03e8bb2098db69210f505a989af3 genkernel-pkg-3.4.tar.bz2 1848904 +MD5 c0a89e47eeaf14ec68f9114f74b16d09 suspend-0.5.tar.gz 163148 +RMD160 8b38fcb18c7f860b4c292a4329f268bce6dc177a suspend-0.5.tar.gz 163148 +SHA256 ab87f1ee097dab98b4d56f98c95779a47a859de81512292743213e6bd5a352f1 suspend-0.5.tar.gz 163148 +MD5 38ed4d9e41c20cf398018928d2c62700 unionfs-1.4.tar.gz 164696 +RMD160 5a19ccfd6edf25f55c48ba1b820347572353c7ba unionfs-1.4.tar.gz 164696 +SHA256 3ff105782aba722b1f263e408fac638245d302a9c8d0f37ac09e5a80caf0126b unionfs-1.4.tar.gz 164696 Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/genkernel-add-arcmsr.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/genkernel-add-arcmsr.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/genkernel-add-arcmsr.patch (revision 24557) @@ -0,0 +1,29 @@ +diff -Nur genkernel-3.4.6/generic/initrd.defaults genkernel-3.4.6.new/generic/initrd.defaults +--- genkernel-3.4.6/generic/initrd.defaults 2006-12-27 09:07:30.000000000 -1000 ++++ genkernel-3.4.6.new/generic/initrd.defaults 2007-07-18 15:55:39.240003000 -1000 +@@ -78,5 +78,5 @@ + + # Only sections that are in by default or those that + # are not module groups need to be defined here... +-HWOPTS='usb firewire keymap cache evms2 sata lvm2 dmraid slowusb fs' +-MY_HWOPTS='usb firewire sata dmraid fs net' ++HWOPTS='usb firewire keymap cache evms2 sata lvm2 dmraid slowusb fs scsi' ++MY_HWOPTS='usb firewire sata dmraid fs net scsi' +diff -Nur genkernel-3.4.6/x86/modules_load genkernel-3.4.6.new/x86/modules_load +--- genkernel-3.4.6/x86/modules_load 2006-12-27 09:07:29.000000000 -1000 ++++ genkernel-3.4.6.new/x86/modules_load 2007-07-18 15:55:19.039999000 -1000 +@@ -1,4 +1,5 @@ + MODULES_SCSI="sd_mod sg sr_mod aic79xx \ ++arcmsr \ + aic7xxx aic7xxx_old BusLogic \ + ncr53c8xx NCR53c406a \ + initio advansys aha1740 aha1542 aha152x \ +diff -Nur genkernel-3.4.6/x86_64/modules_load genkernel-3.4.6.new/x86_64/modules_load +--- genkernel-3.4.6/x86_64/modules_load 2006-12-27 09:07:28.000000000 -1000 ++++ genkernel-3.4.6.new/x86_64/modules_load 2007-07-18 15:55:10.549998000 -1000 +@@ -1,4 +1,5 @@ + MODULES_SCSI="sd_mod sg sr_mod aic79xx \ ++arcmsr \ + aic7xxx aic7xxx_old BusLogic \ + ncr53c8xx NCR53c406a \ + initio advansys aha1740 aha1542 aha152x \ Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/genkernel.bash =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/genkernel.bash (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/genkernel.bash (revision 24557) @@ -0,0 +1,73 @@ +# genkernel (8) completion +# Copyright 2006 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# Written by Aron Griffis + +_genkernel() +{ + declare cur prev genkernel_help actions params + COMPREPLY=() + cur=${COMP_WORDS[COMP_CWORD]} + prev=${COMP_WORDS[COMP_CWORD-1]} + + # extract initial list of params/actions from genkernel --help + genkernel_help=$(command genkernel --help) + actions=( $(<<<"$genkernel_help" sed -n \ + '/^Available Actions:/,/^$/s/^[[:space:]]\+\([^[:space:]]\+\).*/\1/p') ) + params=( $(<<<"$genkernel_help" egrep -oe '--[^[:space:]]{2,}') ) + + # attempt to complete the current parameter based on the list + COMPREPLY=($(compgen -W "${params[*]/=*/=} ${actions[*]}" -- "$cur")) + + # if we don't have a rhs to complete + if [[ ${#COMPREPLY[@]} -gt 1 ]]; then + return + elif [[ ${#COMPREPLY[@]} -eq 0 && $cur != --*=* ]]; then + return + elif [[ ${#COMPREPLY[@]} -eq 1 && $COMPREPLY != --*= ]]; then + # using nospace completion, add an explicit space + COMPREPLY="${COMPREPLY} " + return + fi + + # we have a unique lhs and need to complete the rhs + declare args lhs rhs + if [[ ${#COMPREPLY[@]} -eq 1 ]]; then + lhs=$COMPREPLY + else + lhs=${cur%%=*}= + rhs=${cur#*=} + fi + + # genkernel's help gives clues as to what belongs on the rhs. + # extract the clue for the current parameter + args=" ${params[*]} " + args="${args##* $lhs}" + args="${args%% *}" + + # generate a list of completions for the argument; this replaces args with + # an array of results + args=( $(case $args in + ('<0-5>') compgen -W "$(echo {1..5})" -- "$rhs" ;; + (''|'') compgen -A file -o plusdirs -- "$rhs" ;; + ('') compgen -A directory -S / -- "$rhs" ;; + ('') compgen -G '*.tbz2' -G '*.tar.bz2' -o plusdirs -- "$rhs" ;; + (*) compgen -o bashdefault -- "$rhs" ;; # punt + esac) ) + + # we're using nospace completion to prevent spaces after paths that aren't + # "done" yet. So do some hacking to the args to add spaces after + # non-directories. + declare slash=/ + args=( "${args[@]/%/ }" ) # add space to all + args=( "${args[@]/%$slash /$slash}" ) # remove space from dirs + + # recreate COMPREPLY + if [[ $cur == "$lhs"* ]]; then + COMPREPLY=( "${args[@]}" ) + elif [[ ${#args[@]} -gt 0 ]]; then + COMPREPLY=( "${args[@]/#/$lhs}" ) + fi +} + +complete -o nospace -F _genkernel genkernel Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/suspend-0.5-Makefile.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/suspend-0.5-Makefile.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/files/suspend-0.5-Makefile.patch (revision 24557) @@ -0,0 +1,56 @@ + +From: Daniel Drake + +Index: suspend-0.5/Makefile +=================================================================== +--- suspend-0.5.orig/Makefile ++++ suspend-0.5/Makefile +@@ -1,14 +1,14 @@ + #CONFIG_COMPRESS=yes + #CONFIG_ENCRYPT=yes + #CONFIG_SPLASHY=yes +-#CONFIG_UDEV=yes ++CONFIG_UDEV=yes + + ARCH:=$(shell uname -m) + +-CC_FLAGS=-I/usr/local/include +-LD_FLAGS=-L/usr/local/lib ++CC_FLAGS?=-I/usr/local/include ++LD_FLAGS?=-L/usr/local/lib + +-CFLAGS := -O2 -Wall ++CFLAGS ?= -O2 -Wall + + ifdef CONFIG_COMPRESS + CC_FLAGS += -DCONFIG_COMPRESS +@@ -22,7 +22,7 @@ CC_FLAGS += $(GCRYPT_CC_FLAGS) + LD_FLAGS += $(GCRYPT_LD_FLAGS) + endif + +-SUSPEND_DIR=/usr/local/sbin ++SUSPEND_DIR?=/usr/local/sbin + CONFIG_DIR=/etc + RESUME_DEVICE= + BOOT_DIR=/boot +@@ -31,7 +31,7 @@ S2BOTH=s2both + S2DISK=s2disk + CONFIGFILE=suspend.conf + +-ifndef CONFIG_UDEV ++ifneq ($(CONFIG_UDEV), yes) + SNAPSHOT=$(DESTDIR)/dev/snapshot + endif + +@@ -145,9 +145,9 @@ install-s2disk: $(S2DISK) swap-offset co + if [ -f $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE) ]; then install --mode=644 conf/$(CONFIGFILE) $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE).new; else install --mode=644 conf/$(CONFIGFILE) $(DESTDIR)$(CONFIG_DIR); fi + + install: $(S2DISK) $(S2BOTH) swap-offset conf/$(CONFIGFILE) $(SNAPSHOT) +- install --mode=755 $(S2DISK) $(DESTDIR)$(SUSPEND_DIR) ++ install -D --mode=755 $(S2DISK) $(DESTDIR)$(SUSPEND_DIR)/$(S2DISK) + install --mode=755 $(S2BOTH) $(DESTDIR)$(SUSPEND_DIR) +- if [ -f $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE) ]; then install --mode=644 conf/$(CONFIGFILE) $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE).new; else install --mode=644 conf/$(CONFIGFILE) $(DESTDIR)$(CONFIG_DIR); fi ++ if [ -f $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE) ]; then install --mode=644 conf/$(CONFIGFILE) $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE).new; else install -D --mode=644 conf/$(CONFIGFILE) $(DESTDIR)$(CONFIG_DIR)/$(CONFIGFILE); fi + install --mode=755 s2ram $(DESTDIR)$(SUSPEND_DIR) + install --mode=755 resume $(DESTDIR)$(SUSPEND_DIR) + install --mode=755 swap-offset $(DESTDIR)$(SUSPEND_DIR) Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/genkernel-3.4.6.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/genkernel-3.4.6.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/genkernel-3.4.6.ebuild (revision 24557) @@ -0,0 +1,102 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /var/cvsroot/gentoo-x86/sys-kernel/genkernel/genkernel-3.4.6.ebuild,v 1.3 2007/02/02 22:52:46 wolf31o2 Exp $ + +inherit bash-completion eutils + +VERSION_DMAP='1.02.07' +VERSION_DMRAID='1.0.0.rc13' +VERSION_E2FSPROGS='1.38' +VERSION_LVM2='2.02.05' +VERSION_PKG='3.4' +VERSION_SUSPEND='0.5' +VERSION_UNIONFS='1.4' + +DESCRIPTION="Gentoo autokernel script" +HOMEPAGE="http://www.gentoo.org" +SRC_URI="http://dev.gentoo.org/~wolf31o2/sources/genkernel/${P}.tar.bz2 + http://dev.gentoo.org/~wolf31o2/sources/genkernel/genkernel-pkg-${VERSION_PKG}.tar.bz2 + http://people.redhat.com/~heinzm/sw/dmraid/src/dmraid-${VERSION_DMRAID}.tar.bz2 + ftp://sources.redhat.com/pub/lvm2/old/LVM2.${VERSION_LVM2}.tgz + ftp://sources.redhat.com/pub/dm/old/device-mapper.${VERSION_DMAP}.tgz + ftp://ftp.fsl.cs.sunysb.edu/pub/unionfs/unionfs-${VERSION_UNIONFS}.tar.gz + mirror://sourceforge/e2fsprogs/e2fsprogs-${VERSION_E2FSPROGS}.tar.gz + mirror://sourceforge/suspend/suspend-${VERSION_SUSPEND}.tar.gz" + +LICENSE="GPL-2" +SLOT="0" +# Please don't touch individual KEYWORDS. Since this is maintained/tested by +# Release Engineering, it's easier for us to deal with all arches at once. +#KEYWORDS="~alpha ~amd64 ~arm ~hppa ~ia64 ~mips ~ppc ~ppc64 ~s390 ~sparc ~x86" +KEYWORDS="alpha amd64 arm hppa ia64 mips ppc ppc64 s390 sparc x86" +IUSE="ibm selinux" + +DEPEND="sys-fs/e2fsprogs + selinux? ( sys-libs/libselinux )" +RDEPEND="${DEPEND} app-arch/cpio" + +src_unpack() { + unpack ${P}.tar.bz2 + cd "${S}" + unpack ${PN}-pkg-${VERSION_PKG}.tar.bz2 + cp ${FILESDIR}/suspend-0.5-Makefile.patch pkg + use selinux && sed -i 's/###//g' gen_compile.sh + epatch "${FILESDIR}"/genkernel-add-arcmsr.patch +} + +src_install() { + dodir /etc + cp "${S}"/genkernel.conf ${D}/etc + # This block updates genkernel.conf + sed -i -e "s:VERSION_DMAP:$VERSION_DMAP:" \ + -e "s:VERSION_DMRAID:$VERSION_DMRAID:" \ + -e "s:VERSION_E2FSPROGS:$VERSION_E2FSPROGS:" \ + -e "s:VERSION_LVM2:$VERSION_LVM2:" \ + -e "s:VERSION_UNIONFS:$VERSION_UNIONFS:" \ + -e "s:VERSION_SUSPEND:$VERSION_SUSPEND:" \ + ${D}/etc/genkernel.conf || die "Could not adjust versions" + + dodir /usr/share/genkernel + use ibm && cp "${S}"/ppc64/kernel-2.6-pSeries "${S}"/ppc64/kernel-2.6 || \ + cp "${S}"/ppc64/kernel-2.6.g5 "${S}"/ppc64/kernel-2.6 + cp -Rp "${S}"/* ${D}/usr/share/genkernel + + dodir /usr/bin + dosym /usr/share/genkernel/genkernel /usr/bin/genkernel + + rm ${D}/usr/share/genkernel/genkernel.conf + dodoc README + + doman genkernel.8 + rm genkernel.8 + + cp "${DISTDIR}"/dmraid-${VERSION_DMRAID}.tar.bz2 \ + "${DISTDIR}"/LVM2.${VERSION_LVM2}.tgz \ + "${DISTDIR}"/device-mapper.${VERSION_DMAP}.tgz \ + "${DISTDIR}"/unionfs-${VERSION_UNIONFS}.tar.gz \ + "${DISTDIR}"/e2fsprogs-${VERSION_E2FSPROGS}.tar.gz \ + "${DISTDIR}"/suspend-${VERSION_SUSPEND}.tar.gz \ + ${D}/usr/share/genkernel/pkg + + dobashcompletion ${FILESDIR}/genkernel.bash +} + +pkg_postinst() { + echo + elog 'Documentation is available in the genkernel manual page' + elog 'as well as the following URL:' + echo + elog 'http://www.gentoo.org/doc/en/genkernel.xml' + echo + ewarn "This package is known to not work with reiser4. If you are running" + ewarn "reiser4 and have a problem, do not file a bug. We know it does not" + ewarn "work and we don't plan on fixing it since reiser4 is the one that is" + ewarn "broken in this regard. Try using a sane filesystem like ext3 or" + ewarn "even reiser3." + echo + ewarn "The LUKS support has changed from versions prior to 3.4.4. Now," + ewarn "you use crypt_root=/dev/blah instead of real_root=luks:/dev/blah." + echo + + bash-completion_pkg_postinst +} Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/metadata.xml =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/metadata.xml (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/genkernel/metadata.xml (revision 24557) @@ -0,0 +1,9 @@ + + + + +no-herd + + genkernel@gentoo.org + + Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/gentoo-sources/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/gentoo-sources/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/gentoo-sources/Manifest (revision 24557) @@ -0,0 +1,4 @@ +DIST genpatches-2.6.22-1.base.tar.bz2 1987 RMD160 f5aee41aa9ca56a50845a93d023c38997f664000 SHA1 8703aafdc9e45d32455b88cd24f25c8de246019e SHA256 be058927cfcb447c3f854be74042d3fdcea7fd8b8a6ec445f4b0b37b23b3db81 +DIST genpatches-2.6.22-1.extras.tar.bz2 63706 RMD160 42f3ec0477edb3751068f6fc815aab3f0065d619 SHA1 4715440b92f04466e2fc6f2d5d451b9eb084d62d SHA256 2c1bd61ab9c1c52748f1d70a967efc75d88d41c8f8864a0864754632b32538a8 +DIST linux-2.6.22.tar.bz2 45119878 RMD160 c4b2ca5ee8e71c0f8f1c95f3c9c5f261f2785912 SHA1 39401f195a618f397312d6d84ff9aebe28d99381 SHA256 73c10604c53f1a6ee65ef805293d23903696f8cef864f42d7de9506f0d2ba4c7 +EBUILD gentoo-sources-2.6.22.ebuild 749 RMD160 fb05de4cf1b876a676c8fe53e92a70e2f5f5cd92 SHA1 21ef71bd4dcc43061ce0c43fbc524d50bbd8aa32 SHA256 1cea2ac297e7256dea2d7d7d1e414606bd98a56201ca9059bfe119c6d57e7e61 Index: branches/eam_branches/20090522/portage_overlay/sys-kernel/gentoo-sources/gentoo-sources-2.6.22.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/sys-kernel/gentoo-sources/gentoo-sources-2.6.22.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/sys-kernel/gentoo-sources/gentoo-sources-2.6.22.ebuild (revision 24557) @@ -0,0 +1,22 @@ +# Copyright 1999-2007 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /cvsroot/pan-starrs/project/hw/portage-overlay/sys-kernel/gentoo-sources/gentoo-sources-2.6.22.ebuild,v 1.1 2009/06/23 02:18:44 jhoblitt Exp $ + +ETYPE="sources" +K_WANT_GENPATCHES="base extras" +K_GENPATCHES_VER="1" +inherit kernel-2 +detect_version +detect_arch + +KEYWORDS="~amd64 ~ppc64 ~x86" +HOMEPAGE="http://dev.gentoo.org/~dsd/genpatches" + +DESCRIPTION="Full sources including the Gentoo patchset for the ${KV_MAJOR}.${KV_MINOR} kernel tree" +SRC_URI="${KERNEL_URI} ${GENPATCHES_URI} ${ARCH_URI}" + +pkg_postinst() { + kernel-2_pkg_postinst + einfo "For more info on this patchset, and how to report problems, see:" + einfo "${HOMEPAGE}" +} Index: branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/Manifest =================================================================== --- branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/Manifest (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/Manifest (revision 24557) @@ -0,0 +1,4 @@ +AUX pwauth-gentoo.patch 2658 RMD160 f864529a23da0ee77524e0d8f55521eb7a798eaa SHA1 3c158d4f9770622636d6dff4dbc4fd4b3f7affbc SHA256 9a0f24988ea1453af2f5fa961ed810b0ecab88ebb166fa0993f38081ec4b1b1a +AUX pwauth.pam-include 174 RMD160 f41b0e3d4cc295212fd70863ffaec1a1d34768d8 SHA1 beb89e9116aafe81165b1f46dc1c2a0ee2aa6f31 SHA256 351e55f39d90a4b25369b99fe0444fc6e9eb607fabedf70c8c43cd6192f75067 +DIST pwauth-2.3.5.tar.gz 21234 RMD160 4e2d4e85c49f2edf918b3db0bfa80e452bde6672 SHA1 3d32d427fda801f11adcfeb09886b5c9d8908e65 SHA256 a51870c74e4717c4bc0f233524f8cc19332a900b08c40f54944467a0a8e64928 +EBUILD pwauth-2.3.5.ebuild 1955 RMD160 da0f4556dc1b7c7761bf2b2c8c13cb0ea64df6f5 SHA1 1a67ff88401e1de0c2ca8ff5ff0f5347018ffee6 SHA256 3ec1e5d148d3c6ecf54e63e8c9bb0ac1c45713ab91f8571e9837c3b99eb48859 Index: branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/files/pwauth-gentoo.patch =================================================================== --- branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/files/pwauth-gentoo.patch (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/files/pwauth-gentoo.patch (revision 24557) @@ -0,0 +1,78 @@ +diff -Nurp pwauth-2.3.1.orig/config.h pwauth-2.3.1/config.h +--- pwauth-2.3.1.orig/config.h 2004-09-29 03:07:18.000000000 +0200 ++++ pwauth-2.3.1/config.h 2006-06-06 09:01:37.044516000 +0200 +@@ -106,7 +106,7 @@ + + /* #define SHADOW_NONE /**/ + /* #define SHADOW_BSD /* FreeBSD, NetBSD, OpenBSD, BSDI */ +-#define SHADOW_SUN /* Linux, Solaris, IRIX */ ++/* #define SHADOW_SUN /* Linux, Solaris, IRIX */ + /* #define SHADOW_JFH /**/ + /* #define SHADOW_MDW /**/ + /* #define SHADOW_AIX /* AIX */ +@@ -238,7 +238,7 @@ + * last). + */ + +-#define SERVER_UIDS 72 /* user "nobody" */ ++/* #define SERVER_UIDS 81 /* user "nobody" */ + + + /* If MIN_UNIX_UID is defined to an integer, logins with uid numbers less than +@@ -250,7 +250,7 @@ + * given value will be accepted). + */ + +-#define MIN_UNIX_UID 500 /**/ ++/* #define MIN_UNIX_UID 500 /**/ + + + /* If IGNORE_CASE is defined, the login given is checked in two different +@@ -278,7 +278,7 @@ + * SLEEP_LOCK. SLEEP_TIME defaults to 2 seconds if not defined. + */ + +-#define SLEEP_LOCK "/var/run/pwauth.lock" ++#define SLEEP_LOCK "/var/lock/pwauth" + + + /* If ENV_METHOD is defined, pwauth expects mod_auth_external to be configured +diff -Nurp pwauth-2.3.1.orig/Makefile pwauth-2.3.1/Makefile +--- pwauth-2.3.1.orig/Makefile 2004-09-28 15:40:19.000000000 +0200 ++++ pwauth-2.3.1/Makefile 2006-06-06 09:26:14.620858750 +0200 +@@ -2,32 +2,18 @@ + # system. Most of the configurable stuff is in config.h + # + # CC= an ansi-C compiler. If "cc" doesn't work, try "gcc". +-# LIB= libraries to link in. -lcrypt, -lshadow, -lpam sometimes needed. +-# LOCALFLAGS= compiler flags. Usually -g, -O, and stuff like that. +- +-# Settings for author's system (Redhat 6.1) +-CC=gcc +-LIB= -lcrypt +-LOCALFLAGS= -g +- +-# For PAM on Redhat Linux +-# LIB=-lpam -ldl +- +-# For PAM on Solaris +-# LIB=-lpam ++# LDFLAGS= libraries to link in. -lcrypt, -lshadow, -lpam sometimes needed. + + # -------------------- No User Servicable Parts Below ----------------------- + +-CFLAGS= $(LOCALFLAGS) +- + pwauth: main.o auth_aix.o auth_bsd.o auth_hpux.o auth_mdw.o auth_openbsd.o \ + auth_pam.o auth_sun.o fail_log.o lastlog.o nologin.o snooze.o + $(CC) -o pwauth $(CFLAGS) main.o auth_aix.o auth_bsd.o auth_hpux.o \ + auth_mdw.o auth_openbsd.o auth_pam.o auth_sun.o fail_log.o \ +- lastlog.o nologin.o snooze.o $(LIB) ++ lastlog.o nologin.o snooze.o $(LDFLAGS) + + checkfaillog: checkfaillog.o fail_check.o +- $(CC) -o checkfaillog $(CFLAGS) checkfaillog.o fail_check.o $(LIB) ++ $(CC) -o checkfaillog $(CFLAGS) checkfaillog.o fail_check.o $(LDFLAGS) + + main.o: main.c config.h pwauth.h fail_log.h + auth_aix.o: auth_aix.c config.h pwauth.h Index: branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/files/pwauth.pam-include =================================================================== --- branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/files/pwauth.pam-include (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/files/pwauth.pam-include (revision 24557) @@ -0,0 +1,6 @@ +#%PAM-1.0 +auth required pam_nologin.so +auth include system-auth +account include system-auth +password include system-auth +session include system-auth Index: branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/pwauth-2.3.5.ebuild =================================================================== --- branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/pwauth-2.3.5.ebuild (revision 24557) +++ branches/eam_branches/20090522/portage_overlay/www-apache/pwauth/pwauth-2.3.5.ebuild (revision 24557) @@ -0,0 +1,76 @@ +# Copyright 1999-2008 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 +# $Header: /var/cvsroot/gentoo-x86/www-apache/pwauth/pwauth-2.3.5.ebuild,v 1.1 2008/01/27 18:08:47 hollow Exp $ + +inherit eutils toolchain-funcs pam + +DESCRIPTION="A Unix Web Authenticator" +HOMEPAGE="http://www.unixpapa.com/pwauth/" +SRC_URI="http://www.unixpapa.com/software/${P}.tar.gz" + +LICENSE="Apache-1.1" +SLOT="0" +KEYWORDS="~amd64 ~ppc ~x86" +IUSE="faillog pam ignore-case domain-aware" + +DEPEND="pam? ( virtual/pam )" + +pkg_setup() { + local OPTS + + einfo "You can configure various build time options with ENV variables:" + einfo + einfo " PWAUTH_FAILLOG Path to logfile for login failures" + einfo " (default: /var/log/pwauth.log)" + einfo " PWAUTH_SERVERUIDS Comma seperated list of UIDs allowed to run pwauth" + einfo " (default: 81)" + einfo " PWAUTH_MINUID Minimum UID for which authentication will succeed" + einfo " (default: 100)" + einfo + + PWAUTH_FAILLOG="${PWAUTH_FAILLOG:-/var/log/pwauth.log}" + PWAUTH_SERVERUIDS="${PWAUTH_SERVERUIDS:-81}" + PWAUTH_MINUID="${PWAUTH_MINUID:-100}" + + OPTS="${OPTS} -DSERVER_UIDS=${PWAUTH_SERVERUIDS}" + OPTS="${OPTS} -DMIN_UNIX_UID=${PWAUTH_MINUID}" + + if use faillog; then + OPTS="${OPTS} -DFAILLOG_PWAUTH" + OPTS="${OPTS} -DPATH_FAILLOG=\"\\\"${PWAUTH_FAILLOG}\\\"\"" + fi + + if use pam; then + OPTS="${OPTS} -DPAM" + LDFLAGS="-lpam" + else + OPTS="${OPTS} -DSHADOW_SUN" + LDFLAGS="-lcrypt" + fi + + if use ignore-case; then + OPTS="${OPTS} -DIGNORE_CASE" + fi + + if use domain-aware; then + OPTS="${OPTS} -DOMAIN_AWARE" + fi + + CC=$(tc-getCC) + CFLAGS="${CFLAGS} ${OPTS}" +} + +src_unpack() { + unpack ${A} + cd "${S}" + epatch "${FILESDIR}"/pwauth-gentoo.patch +} + +src_install() { + dosbin pwauth unixgroup + fperms 4755 /usr/sbin/pwauth + + use pam && newpamd "${FILESDIR}"/pwauth.pam-include pwauth + + dodoc CHANGES FORM_AUTH INSTALL README +} Index: branches/eam_branches/20090522/ppImage/notes.txt =================================================================== --- branches/eam_branches/20090522/ppImage/notes.txt (revision 24529) +++ branches/eam_branches/20090522/ppImage/notes.txt (revision 24557) @@ -1,23 +1,15 @@ -ppImage pmFPAfiles: +20090611 : adjusting things to allow the readnoise to be an image map for each cell - pmFPAfile *input = pmFPAfileDefineFromArgs (&status, config, "PPIMAGE.INPUT", "INPUT"); - pmFPAfile *inputMask = pmFPAfileBindFromArgs(&status, input, config, "PPIMAGE.INPUT.MASK", "INPUT.MASK"); - pmFPAfile *inputWeight = pmFPAfileBindFromArgs(&status, input, config, "PPIMAGE.INPUT.WEIGHT", "INPUT.WEIGHT"); - pmFPAfile *output = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.OUTPUT"); - pmFPAfile *outMask = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.OUTPUT.MASK"); - pmFPAfile *outWeight = pmFPAfileDefineOutput(config, input->fpa, "PPIMAGE.OUTPUT.WEIGHT"); - pmFPAfile *chipImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PPIMAGE.CHIP"); - pmFPAfile *chipMask = pmFPAfileDefineOutput(config, chipImage->fpa, "PPIMAGE.CHIP.MASK"); - pmFPAfile *chipWeight = pmFPAfileDefineOutput(config, chipImage->fpa, "PPIMAGE.CHIP.WEIGHT"); - pmFPAfile *byFPA1 = pmFPAfileDefineFPAMosaic(config, input->fpa, "PPIMAGE.OUTPUT.FPA1"); - pmFPAfile *byFPA2 = pmFPAfileDefineFPAMosaic(config, input->fpa, "PPIMAGE.OUTPUT.FPA2"); + * pmReadoutSetVaraince : when variance is created, we are suppling a single CELL.READNOISE value -- change this to an optional image (carried through concepts?) - pmFPAfile *psphotInput = pmFPAfileDefineFromFPA (config, chipImage->fpa, 1, 1, "PSPHOT.INPUT"); - pmFPAfile *psphotOutput = psMetadataLookupPtr (&status, config->files, "PSPHOT.OUTPUT"); - pmFPAfile *psastroInput = pmFPAfileDefineInput (config, psphotOutput->fpa, "PSASTRO.INPUT"); + I need to: - pmFPAfile *bin1 = pmFPAfileDefineFromFPA (config, chipImage->fpa, options->xBin1, options->yBin1, "PPIMAGE.BIN1"); - pmFPAfile *bin2 = pmFPAfileDefineFromFPA (config, chipImage->fpa, options->xBin2, options->yBin2, "PPIMAGE.BIN2"); - pmFPAfile *jpg1 = pmFPAfileDefineOutput (config, byFPA1->fpa, "PPIMAGE.JPEG1"); - pmFPAfile *jpg2 = pmFPAfileDefineOutput (config, byFPA2->fpa, "PPIMAGE.JPEG2"); + * define a format for the readnoise map (image map in SPLIT/MEF format?) + * load the readnoise map + + * supply the map for each cell to the pmReadoutSetVariance (pointer to ppImageMap?) + pmReadoutSetVariance -> add psImageMap to API (also pmReadoutGenerateVariance) + + * add recipe information to define the source of the READNOISE info. + Index: branches/eam_branches/20090522/ppImage/src/ppImage.h =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImage.h (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImage.h (revision 24557) @@ -31,4 +31,5 @@ bool doNonLin; // Non-linearity correction bool doOverscan; // Overscan subtraction + bool doNoiseMap; // Bias subtraction bool doBias; // Bias subtraction bool doDark; // Dark subtraction Index: branches/eam_branches/20090522/ppImage/src/ppImageArguments.c =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImageArguments.c (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImageArguments.c (revision 24557) @@ -17,4 +17,5 @@ fprintf(stderr, "\n"); fprintf(stderr, "Input options (single file / file list):\n"); + fprintf(stderr, "\t-noisemap/-noisemaplist: Noise Map image.\n"); fprintf(stderr, "\t-bias/-biaslist: Bias image.\n"); fprintf(stderr, "\t-dark/-darklist: Dark image.\n"); @@ -112,4 +113,5 @@ // if these command-line options are supplied, load the file name lists into config->arguments // override any configuration-specified source for these files + pmConfigFileSetsMD (config->arguments, &argc, argv, "NOISEMAP", "-noisemap", "-noisemaplist"); pmConfigFileSetsMD (config->arguments, &argc, argv, "BIAS", "-bias", "-biaslist"); pmConfigFileSetsMD (config->arguments, &argc, argv, "DARK", "-dark", "-darklist"); Index: branches/eam_branches/20090522/ppImage/src/ppImageDetrendFree.c =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImageDetrendFree.c (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImageDetrendFree.c (revision 24557) @@ -8,4 +8,5 @@ static char *detrendTypes[] = { "PPIMAGE.MASK", + "PPIMAGE.NOISEMAP", "PPIMAGE.BIAS", "PPIMAGE.DARK", Index: branches/eam_branches/20090522/ppImage/src/ppImageDetrendReadout.c =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImageDetrendReadout.c (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImageDetrendReadout.c (revision 24557) @@ -64,5 +64,27 @@ if (options->doVarianceBuild) { // create the target mask and variance images - pmReadoutGenerateVariance(input, true); + psImage *noiseImage = NULL; + if (options->doNoiseMap) { + // XXX convert the noiseMap image to a binned image + pmReadout *noiseMap = NULL; + noiseMap = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.NOISEMAP"); + noiseImage = psImageCopy (NULL, input->image, PS_TYPE_F32); + psImageInit (noiseImage, 0.0); + + // XXX this works, but is not really quite right: the model shoud include the + // offset information, we are not really getting exactly the right mapping from the + // original file. + psImageBinning *binning = psImageBinningAlloc(); + binning->nXruff = noiseMap->image->numCols; + binning->nYruff = noiseMap->image->numRows; + binning->nXfine = input->image->numCols; + binning->nYfine = input->image->numRows; + psImageBinningSetScale(binning, PS_IMAGE_BINNING_LEFT); + + psImageUnbin (noiseImage, noiseMap->image, binning); + psFree (binning); + } + pmReadoutGenerateVariance(input, noiseImage, true); + psFree (noiseImage); } Index: branches/eam_branches/20090522/ppImage/src/ppImageDetrendRecord.c =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImageDetrendRecord.c (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImageDetrendRecord.c (revision 24557) @@ -67,12 +67,11 @@ psMetadataAddMetadata(cell->analysis, PS_LIST_TAIL, "DETREND", 0, "Detrend information", detrend); - detrendRecord(options->doMask, detrend, config, view, "PPIMAGE.MASK", "DETREND.MASK", "Mask filename"); - detrendRecord(options->doBias, detrend, config, view, "PPIMAGE.BIAS", "DETREND.BIAS", "Bias filename"); - detrendRecord(options->doDark, detrend, config, view, "PPIMAGE.DARK", "DETREND.DARK", "Dark filename"); - detrendRecord(options->doShutter, detrend, config, view, "PPIMAGE.SHUTTER", "DETREND.SHUTTER", - "Shutter correction filename"); - detrendRecord(options->doFlat, detrend, config, view, "PPIMAGE.FLAT", "DETREND.FLAT", "Flat filename"); - detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE", - "Fringe filename"); + detrendRecord(options->doMask, detrend, config, view, "PPIMAGE.MASK", "DETREND.MASK", "Mask filename"); + detrendRecord(options->doNoiseMap, detrend, config, view, "PPIMAGE.NOISEMAP", "DETREND.NOISEMAP", "Noise Map filename"); + detrendRecord(options->doBias, detrend, config, view, "PPIMAGE.BIAS", "DETREND.BIAS", "Bias filename"); + detrendRecord(options->doDark, detrend, config, view, "PPIMAGE.DARK", "DETREND.DARK", "Dark filename"); + detrendRecord(options->doShutter, detrend, config, view, "PPIMAGE.SHUTTER", "DETREND.SHUTTER", "Shutter correction filename"); + detrendRecord(options->doFlat, detrend, config, view, "PPIMAGE.FLAT", "DETREND.FLAT", "Flat filename"); + detrendRecord(options->doFringe, detrend, config, view, "PPIMAGE.FRINGE", "DETREND.FRINGE", "Fringe filename"); psFree (detrend); Index: branches/eam_branches/20090522/ppImage/src/ppImageOptions.c =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImageOptions.c (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImageOptions.c (revision 24557) @@ -25,4 +25,5 @@ options->doNonLin = false; // Non-linearity correction options->doOverscan = false; // Overscan subtraction + options->doNoiseMap = false; // Apply Read Noise Map options->doBias = false; // Bias subtraction options->doDark = false; // Dark subtraction @@ -217,4 +218,5 @@ options->doCrosstalkCorrect = psMetadataLookupBool(NULL, recipe, "CROSSTALK.CORRECT"); + options->doNoiseMap = psMetadataLookupBool(NULL, recipe, "NOISEMAP"); options->doBias = psMetadataLookupBool(NULL, recipe, "BIAS"); options->doDark = psMetadataLookupBool(NULL, recipe, "DARK"); @@ -278,5 +280,5 @@ // even if not requested explicitly, if any of these are set, build an internal mask and variance: - if (options->doBias || options->doOverscan || options->doDark || options->doShutter || options->doFlat || + if (options->doNoiseMap || options->doBias || options->doOverscan || options->doDark || options->doShutter || options->doFlat || options->doPhotom) { options->doMaskBuild = true; Index: branches/eam_branches/20090522/ppImage/src/ppImageParseCamera.c =================================================================== --- branches/eam_branches/20090522/ppImage/src/ppImageParseCamera.c (revision 24529) +++ branches/eam_branches/20090522/ppImage/src/ppImageParseCamera.c (revision 24557) @@ -25,4 +25,12 @@ // otherwise they revert to the config information // not all input or output images are used in a given recipe + if (options->doNoiseMap) { + if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.NOISEMAP", "NOISEMAP", + PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NOISEMAP)) { + psError(PS_ERR_IO, false, "Can't find a noise map image source"); + psFree(options); + return NULL; + } + } if (options->doBias) { if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.BIAS", "BIAS", @@ -48,30 +56,4 @@ return NULL; } - -#if 0 - // I think this is now done automatically in the pmFPAfileDefine and pmFPAfileIOChecks -- PAP. - - // XXX have ppImageDefineFile return the pmFPAfile? - pmFPAfile *mask = psMetadataLookupPtr(&status, config->files, "PPIMAGE.MASK"); - psAssert(mask, "Just defined the mask!"); - - // Need to read the names of bit masks from the mask header and set them in the - // recipe. If we are loading this from the detrend db, this action will happen - // when the file is resolved. - if (!mask->detrend) { - // XXX need to load the mask bit names from one of the headers - // this grabs the first available hdu : no guarantee that it will be valid, though - pmHDU *hdu = pmHDUGetFirst(mask->fpa); - if (!hdu) { - psError(PS_ERR_IO, true, "no valid HDU for PPIMAGE.INPUT.MASK"); - return NULL; - } - // XXX is this consistent with the pmConfigMaskReadHeader call above? - if (!pmConfigMaskReadHeader(config, hdu->header)) { - psError(PS_ERR_IO, false, "error in mask bits"); - return NULL; - } - } -#endif } if (options->doShutter) { Index: branches/eam_branches/20090522/ppMops/Makefile.am =================================================================== --- branches/eam_branches/20090522/ppMops/Makefile.am (revision 24557) +++ branches/eam_branches/20090522/ppMops/Makefile.am (revision 24557) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *.pyc *~ core core.* Index: branches/eam_branches/20090522/ppMops/autogen.sh =================================================================== --- branches/eam_branches/20090522/ppMops/autogen.sh (revision 24557) +++ branches/eam_branches/20090522/ppMops/autogen.sh (revision 24557) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppMops +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOLIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: branches/eam_branches/20090522/ppMops/configure.ac =================================================================== --- branches/eam_branches/20090522/ppMops/configure.ac (revision 24557) +++ branches/eam_branches/20090522/ppMops/configure.ac (revision 24557) @@ -0,0 +1,42 @@ +dnl Process this file with autoconf to produce a configure script. +AC_PREREQ(2.61) + +AC_INIT([ppMops], [0.1.1], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.0.0]) + +AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing]) +if test "$ERRORCODES" = "missing" ; then + AC_MSG_ERROR([psParseErrorCodes is required]) +fi + +dnl Set CFLAGS for build +IPP_STDOPTS +CFLAGS="${CFLAGS} -Wall -Werror" + +IPP_VERSION + +AC_SUBST([PPMOPS_CFLAGS]) +AC_SUBST([PPMOPS_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile +]) + +AC_OUTPUT Index: branches/eam_branches/20090522/ppMops/src/Makefile.am =================================================================== --- branches/eam_branches/20090522/ppMops/src/Makefile.am (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/Makefile.am (revision 24557) @@ -0,0 +1,41 @@ +bin_PROGRAMS = ppMops + +if HAVE_SVNVERSION +PPMOPS_VERSION=`$(SVNVERSION) ..` +else +PPMOPS_VERSION="UNKNOWN" +endif + +if HAVE_SVN +PPMOPS_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'` +PPMOPS_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'` +else +PPMOPS_BRANCH="UNKNOWN" +PPMOPS_SOURCE="UNKNOWN" +endif + +# Force recompilation of ppMopsVersion.c, since it gets the version information +ppMopsVersion.c: ppMopsVersionDefinitions.h +ppMopsVersionDefinitions.h: ppMopsVersionDefinitions.h.in FORCE + -$(RM) ppMopsVersionDefinitions.h + $(SED) -e "s|@PPMOPS_VERSION@|\"$(PPMOPS_VERSION)\"|" -e "s|@PPMOPS_BRANCH@|\"$(PPMOPS_BRANCH)\"|" -e "s|@PPMOPS_SOURCE@|\"$(PPMOPS_SOURCE)\"|" ppMopsVersionDefinitions.h.in > ppMopsVersionDefinitions.h +FORCE: ; + +ppMops_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(PPMOPS_CFLAGS) +ppMops_LDFLAGS = $(PSLIB_LIBS) $(PPMOPS_LIBS) + +ppMops_SOURCES = \ + ppMops.c \ + ppMopsVersion.c \ + ppMopsData.c + +noinst_HEADERS = \ + ppMops.h + + +clean-local: + -rm -f TAGS + +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: branches/eam_branches/20090522/ppMops/src/ppMops.c =================================================================== --- branches/eam_branches/20090522/ppMops/src/ppMops.c (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/ppMops.c (revision 24557) @@ -0,0 +1,210 @@ +#include +#include + +#include "ppMops.h" + +int main(int argc, char *argv[]) +{ + if (argc != 4) { + fprintf(stderr, "Insufficient arguments.\n"); + fprintf(stderr, "Usage: %s DETECTIONS ZP OUTPUT\n", argv[0]); + exit(PS_EXIT_CONFIG_ERROR); + } + + ppMopsData *data = ppMopsDataAlloc(); // Configuration data + data->detections = psStringCopy(argv[1]); + data->zp = atof(argv[2]); + data->output = psStringCopy(argv[3]); + + if (!isfinite(data->zp)) { + psErrorStackPrint(stderr, "Zero point is unknown\n"); + exit(PS_EXIT_CONFIG_ERROR); + } + + psArray *detections = NULL; // Detections + psMetadata *header = NULL; // Header for detections + { + psFits *fits = psFitsOpen(data->detections, "r"); // FITS file + header = psFitsReadHeader(NULL, fits); + if (!header) { + psErrorStackPrint(stderr, "Unable to read header"); + psFitsClose(fits); + psFree(data); + exit(PS_EXIT_DATA_ERROR); + } + if (!psFitsMoveExtName(fits, IN_EXTNAME)) { + psErrorStackPrint(stderr, "Unable to move to extension %s", IN_EXTNAME); + psFitsClose(fits); + psFree(header); + psFree(data); + exit(PS_EXIT_DATA_ERROR); + } + detections = psFitsReadTable(fits); + psFitsClose(fits); + if (!detections) { + psErrorStackPrint(stderr, "Unable to read detections"); + psFree(data); + psFree(header); + exit(PS_EXIT_DATA_ERROR); + } + } + + // Translate the columns + int numIn = detections->n; // Number of rows in input + int numOut = 0; // Number of rows in output + psArray *output = psArrayAllocEmpty(numIn); // Output table + double plateScale = 0.0; // Average plate scale + for (int i = 0; i < numIn; i++) { + psMetadata *inRow = detections->data[i]; // Input row + + double ra = psMetadataLookupF64(NULL, inRow, "RA_PSF"); + double dec = psMetadataLookupF64(NULL, inRow, "DEC_PSF"); + double mag = psMetadataLookupF64(NULL, inRow, "PSF_INST_MAG") + data->zp; + double magErr = psMetadataLookupF64(NULL, inRow, "PSF_INST_MAG_SIG"); + double ext = psMetadataLookupF64(NULL, inRow, "EXT_NSIGMA"); + double xErr = psMetadataLookupF64(NULL, inRow, "X_PSF_SIG"); + double yErr = psMetadataLookupF64(NULL, inRow, "Y_PSF_SIG"); + double scale = psMetadataLookupF64(NULL, inRow, "PLTSCALE"); + double angle = psMetadataLookupF64(NULL, inRow, "POSANGLE"); + psU32 flags = psMetadataLookupU32(NULL, inRow, "FLAGS"); + + if (!isfinite(mag) || !isfinite(magErr) || (flags & SOURCE_MASK) || + ((flags & PM_SOURCE_MODE_DEFECT) && !(flags & PM_SOURCE_MODE_MOMENTS_FAILURE)) + ) { + // DEFECT can be due to bad moments + continue; + } + + psMetadata *outRow = output->data[numOut] = psMetadataAlloc(); // Output row + + numOut++; + plateScale += scale; + + // XXX Not at all sure I've got the angles around the right way here... + double cosAngle = cos(angle), sinAngle = sin(angle); + double raErr = scale * sqrt(PS_SQR(cosAngle) * PS_SQR(xErr) + PS_SQR(sinAngle) * PS_SQR(yErr)); + double decErr = scale * sqrt(PS_SQR(sinAngle) * PS_SQR(xErr) + PS_SQR(cosAngle) * PS_SQR(yErr)); + + psMetadataAddF64(outRow, PS_LIST_TAIL, "RA_DEG", 0, "Right ascension (degrees)", ra); + psMetadataAddF64(outRow, PS_LIST_TAIL, "DEC_DEG", 0, "Declination (degrees)", dec); + psMetadataAddF64(outRow, PS_LIST_TAIL, "RA_SIG", 0, "Right ascension error (degrees)", raErr); + psMetadataAddF64(outRow, PS_LIST_TAIL, "DEC_SIG", 0, "Declination error (degrees)", decErr); + psMetadataAddF64(outRow, PS_LIST_TAIL, "MAG", 0, "Magnitude", mag); + psMetadataAddF64(outRow, PS_LIST_TAIL, "MAG_SIG", 0, "Magnitude error", magErr); + psMetadataAddU32(outRow, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", flags); + + // The below need fixing + psMetadataAddF64(outRow, PS_LIST_TAIL, "STARPSF", 0, "EXT_NSIGMA", ext); + psMetadataAddF64(outRow, PS_LIST_TAIL, "ANG", 0, "Position angle of trail (degrees)", 0.0); + psMetadataAddF64(outRow, PS_LIST_TAIL, "ANG_SIG", 0, "Position angle error (degrees)", 0.0); + psMetadataAddF64(outRow, PS_LIST_TAIL, "LEN", 0, "Length of trail (arcsec)", 0.0); + psMetadataAddF64(outRow, PS_LIST_TAIL, "LEN_SIG", 0, "Length error (arcsec)", 0.0); + } + output->n = numOut; + plateScale /= numOut; + psFree(detections); + + // Translate the header + psMetadata *outHeader = psMetadataAlloc(); // Output header + ppMopsVersionHeader(outHeader); + { + const char *ra = psMetadataLookupStr(NULL, header, "FPA.RA"); + const char *dec = psMetadataLookupStr(NULL, header, "FPA.DEC"); + const char *filter = psMetadataLookupStr(NULL, header, "FPA.FILTER"); + float airmass = psMetadataLookupF32(NULL, header, "FPA.AIRMASS"); + float exptime = psMetadataLookupF32(NULL, header, "EXPTIME"); + double angle = psMetadataLookupF64(NULL, header, "FPA.POSANGLE"); + double alt = psMetadataLookupF64(NULL, header, "FPA.ALT"); + double az = psMetadataLookupF64(NULL, header, "FPA.AZ"); + int imageid = psMetadataLookupS32(NULL, header, "IMAGEID"); + double mjd = psMetadataLookupF64(NULL, header, "MJD-OBS") + exptime / 2.0 / 3600 / 24; + + float psf = plateScale * 0.5 * (psMetadataLookupF32(NULL, header, "FWHM_MAJ") + + psMetadataLookupF32(NULL, header, "FWHM_MIN")); + + // XXX This is wrong + int fpaid = psMetadataLookupS32(NULL, header, "IMAGEID"); + + + psMetadataAddStr(outHeader, PS_LIST_TAIL, "RA", 0, "Right ascension of boresight", ra); + psMetadataAddStr(outHeader, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", dec); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", airmass); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", mjd); + psMetadataAddStr(outHeader, PS_LIST_TAIL, "FILTER", 0, "Filter name", filter); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", exptime); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "ROTANGLE", 0, "Rotator position angle", angle); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", alt); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", az); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "DIFFIMID", 0, "Difference image identifier", imageid); + psMetadataAddF64(outHeader, PS_LIST_TAIL, "FPA_ID", 0, "Exposure identifier", fpaid); + psMetadataAddStr(outHeader, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "STARPSF", 0, "Stellar PSF (arcsec)", psf); + + // These are completely fake + psMetadataAddF32(outHeader, PS_LIST_TAIL, "LIMITMAG", 0, "Limiting magnitude (FAKE)", 25.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE1", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE2", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE3", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE4", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE5", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE6", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE7", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE8", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE9", 0, "Detection efficiency (FAKE)", 0.0); + psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE10", 0, "Detection efficiency (FAKE)", 0.0); + } + psFree(header); + + // Write the new table + { + psFits *fits = psFitsOpen(data->output, "w"); // FITS file + if (!fits) { + psErrorStackPrint(stderr, "Unable to open %s for writing", data->output); + psFree(outHeader); + psFree(output); + psFree(data); + exit(PS_EXIT_SYS_ERROR); + } + + if (numOut == 0) { + // Write dummy table + psMetadata *outRow = psMetadataAlloc(); // Dummy output row + psMetadataAddF64(outRow, PS_LIST_TAIL, "RA_DEG", 0, "Right ascension (degrees)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "DEC_DEG", 0, "Declination (degrees)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "RA_SIG", 0, "Right ascension error (degrees)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "DEC_SIG", 0, "Declination error (degrees)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "MAG", 0, "Magnitude", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "MAG_SIG", 0, "Magnitude error", NAN); + psMetadataAddU32(outRow, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", 0); + + // The below need fixing + psMetadataAddF64(outRow, PS_LIST_TAIL, "STARPSF", 0, "EXT_NSIGMA", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "ANG", 0, "Position angle of trail (degrees)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "ANG_SIG", 0, "Position angle error (degrees)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "LEN", 0, "Length of trail (arcsec)", NAN); + psMetadataAddF64(outRow, PS_LIST_TAIL, "LEN_SIG", 0, "Length error (arcsec)", NAN); + if (!psFitsWriteTableEmpty(fits, outHeader, outRow, OUT_EXTNAME)) { + psErrorStackPrint(stderr, "Unable to write table."); + psFree(outHeader); + psFree(output); + psFree(outRow); + psFree(data); + exit(PS_EXIT_SYS_ERROR); + } + psFree(outRow); + } else if (!psFitsWriteTable(fits, outHeader, output, OUT_EXTNAME)) { + psErrorStackPrint(stderr, "Unable to write table."); + psFree(outHeader); + psFree(output); + psFree(data); + exit(PS_EXIT_SYS_ERROR); + } + psFitsClose(fits); + } + + psFree(outHeader); + psFree(output); + psFree(data); + + return PS_EXIT_SUCCESS; +} Index: branches/eam_branches/20090522/ppMops/src/ppMops.h =================================================================== --- branches/eam_branches/20090522/ppMops/src/ppMops.h (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/ppMops.h (revision 24557) @@ -0,0 +1,41 @@ +#ifndef PP_MOPS_H +#define PP_MOPS_H + +#include +#include + +#define IN_EXTNAME "SkyChip.psf" // Extension name for data in input +#define OBSERVATORY_CODE "F51" // IAU Observatory Code +#define OUT_EXTNAME "MOPS_TRANSIENT_DETECTIONS" // Extension name for data in output +#define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_SATURATED | \ + PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_SKY_FAILURE) // Flags to exclude + + +// Configuration data +typedef struct { + psString detections; // Detections filename + float zp; // Magnitude zero point + psString output; // Output filename +} ppMopsData; + +// Allocator +ppMopsData *ppMopsDataAlloc(void); + + + +/// Return version +psString ppMopsVersion(void); + +/// Return source +psString ppMopsSource(void); + +/// Return detailed version information +psString ppMopsVersionLong(void); + +/// Put version into header +bool ppMopsVersionHeader(psMetadata *header); + +/// Print version information +void ppMopsVersionPrint(void); + +#endif Index: branches/eam_branches/20090522/ppMops/src/ppMopsData.c =================================================================== --- branches/eam_branches/20090522/ppMops/src/ppMopsData.c (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/ppMopsData.c (revision 24557) @@ -0,0 +1,29 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppMops.h" + +static void mopsDataFree(ppMopsData *data) +{ + psFree(data->detections); + psFree(data->output); + return; +} + +ppMopsData *ppMopsDataAlloc(void) +{ + ppMopsData *data = psAlloc(sizeof(ppMopsData)); // Data to return + psMemSetDeallocator(data, (psFreeFunc)mopsDataFree); + + data->detections = NULL; + data->zp = NAN; + data->output = NULL; + + return data; +} + + Index: branches/eam_branches/20090522/ppMops/src/ppMopsVersion.c =================================================================== --- branches/eam_branches/20090522/ppMops/src/ppMopsVersion.c (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/ppMopsVersion.c (revision 24557) @@ -0,0 +1,100 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "ppMops.h" +#include "ppMopsVersionDefinitions.h" + +#ifndef PPMOPS_VERSION +#error "PPMOPS_VERSION is not set" +#endif +#ifndef PPMOPS_BRANCH +#error "PPMOPS_BRANCH is not set" +#endif +#ifndef PPMOPS_SOURCE +#error "PPMOPS_SOURCE is not set" +#endif + +psString ppMopsVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PPMOPS_BRANCH, PPMOPS_VERSION); + return value; +} + +psString ppMopsSource(void) +{ + return psStringCopy(PPMOPS_SOURCE); +} + +psString ppMopsVersionLong(void) +{ + psString version = ppMopsVersion(); // Version, to return + psString source = ppMopsSource(); // Source + + psStringPrepend(&version, "ppMops "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + + return version; +}; + + +bool ppMopsVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "ppMops at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + + psString version = ppMopsVersion(); // Software version + psString source = ppMopsSource(); // Software source + + psStringPrepend(&version, "ppMops version: "); + psStringPrepend(&source, "ppMops source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + + return true; +} + +void ppMopsVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("ppMops", PS_LOG_INFO, "ppMops at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString ppMops = ppMopsVersionLong(); // ppMops version + + psLogMsg("ppMops", PS_LOG_INFO, "%s", pslib); + psLogMsg("ppMops", PS_LOG_INFO, "%s", ppMops); + + psFree(pslib); + psFree(ppMops); + + return; +} Index: branches/eam_branches/20090522/ppMops/src/ppMopsVersionDefinitions.h.in =================================================================== --- branches/eam_branches/20090522/ppMops/src/ppMopsVersionDefinitions.h.in (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/ppMopsVersionDefinitions.h.in (revision 24557) @@ -0,0 +1,8 @@ +#ifndef PPMOPS_VERSION_DEFINITIONS_H +#define PPMOPS_VERSION_DEFINITIONS_H + +#define PPMOPS_VERSION @PPMOPS_VERSION@ // SVN version +#define PPMOPS_BRANCH @PPMOPS_BRANCH@ // SVN branch +#define PPMOPS_SOURCE @PPMOPS_SOURCE@ // SVN source + +#endif Index: branches/eam_branches/20090522/ppMops/src/ppSubVersion.c =================================================================== --- branches/eam_branches/20090522/ppMops/src/ppSubVersion.c (revision 24557) +++ branches/eam_branches/20090522/ppMops/src/ppSubVersion.c (revision 24557) @@ -0,0 +1,127 @@ +/** @file ppSubVersion.c + * + * @brief + * + * @ingroup ppSub + * + * @author IfA + * @version $Revision: 1.4 $ $Name: not supported by cvs2svn $ + * @date $Date: 2009-02-18 00:31:20 $ + * Copyright 2009 Institute for Astronomy, University of Hawaii + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppSub.h" +#include "ppSubVersionDefinitions.h" + +#ifndef PPSUB_VERSION +#error "PPSUB_VERSION is not set" +#endif +#ifndef PPSUB_BRANCH +#error "PPSUB_BRANCH is not set" +#endif +#ifndef PPSUB_SOURCE +#error "PPSUB_SOURCE is not set" +#endif + +psString ppSubVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PPSUB_BRANCH, PPSUB_VERSION); + return value; +} + +psString ppSubSource(void) +{ + return psStringCopy(PPSUB_SOURCE); +} + +psString ppSubVersionLong(void) +{ + psString version = ppSubVersion(); // Version, to return + psString source = ppSubSource(); // Source + + psStringPrepend(&version, "ppSub "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + + return version; +}; + + +bool ppSubVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "ppSub at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + psModulesVersionHeader(header); + psphotVersionHeader(header); + ppStatsVersionHeader(header); + + psString version = ppSubVersion(); // Software version + psString source = ppSubSource(); // Software source + + psStringPrepend(&version, "ppSub version: "); + psStringPrepend(&source, "ppSub source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + + return true; +} + +void ppSubVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("ppSub", PS_LOG_INFO, "ppSub at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString psphot = psphotVersionLong(); // psphot version + psString ppStats = ppStatsVersionLong(); // ppStats version + psString ppSub = ppSubVersionLong(); // ppSub version + + psLogMsg("ppSub", PS_LOG_INFO, "%s", pslib); + psLogMsg("ppSub", PS_LOG_INFO, "%s", psmodules); + psLogMsg("ppSub", PS_LOG_INFO, "%s", psphot); + psLogMsg("ppSub", PS_LOG_INFO, "%s", ppStats); + psLogMsg("ppSub", PS_LOG_INFO, "%s", ppSub); + + psFree(pslib); + psFree(psmodules); + psFree(psphot); + psFree(ppStats); + psFree(ppSub); + + return; +} Index: branches/eam_branches/20090522/ppNoiseMap/Doxyfile.in =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/Doxyfile.in (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/Doxyfile.in (revision 24557) @@ -0,0 +1,1310 @@ +# Doxyfile 1.5.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project +# +# All text after a hash (#) is considered a comment and will be ignored +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" ") + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file that +# follow. The default is UTF-8 which is also the encoding used for all text before +# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into +# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of +# possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = ppNoiseMap + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = ipp-2.9 + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = ./docs + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, +# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, +# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, +# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = NO + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful is your file systems +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the DETAILS_AT_TOP tag is set to YES then Doxygen +# will output the detailed description near the top, like JavaDoc. +# If set to NO, the detailed description appears after the member +# documentation. + +DETAILS_AT_TOP = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for Java. +# For instance, namespaces will be presented as packages, qualified scopes +# will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to +# include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is +# documented as struct with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code where the coding convention is that all structs are +# typedef'ed and only the typedef is referenced never the struct's name. + +TYPEDEF_HIDES_STRUCT = NO + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be extracted +# and appear in the documentation as a namespace called 'anonymous_namespace{file}', +# where file will be replaced with the base name of the file that contains the anonymous +# namespace. By default anonymous namespace are hidden. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or define consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and defines in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from the +# version control system). Doxygen will invoke the program by executing (via +# popen()) the command , where is the value of +# the FILE_VERSION_FILTER tag, and is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = YES + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be abled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text " + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = ./src + +# This tag can be used to specify the character encoding of the source files that +# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default +# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. +# See http://www.gnu.org/software/libiconv for the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx +# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90 + +FILE_PATTERNS = *.h + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix filesystem feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the output. +# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, +# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command , where +# is the value of the INPUT_FILTER tag, and is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER +# is applied to all files. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH +# then you must also enable this option. If you don't then doxygen will produce +# a warning and turn it on anyway + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES (the default) +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES (the default) +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. Otherwise they will link to the documentstion. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = NO + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# This tag can be used to set the number of enum values (range [1..20]) +# that doxygen will group on one line in the generated HTML documentation. + +ENUM_VALUES_PER_LINE = 4 + +# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be +# generated containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, +# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are +# probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = NO + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, a4wide, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4wide + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = NO + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = NO + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = YES + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. This is useful +# if you want to understand what is going on. On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# in the INCLUDE_PATH (see below) will be search if a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all function-like macros that are alone +# on a line, have an all uppercase name, and do not end with a semicolon. Such +# function macros are typically used for boiler-plate code, and will confuse +# the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option is superseded by the HAVE_DOT option below. This is only a +# fallback. It is recommended to install and use dot, since it yields more +# powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to +# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to +# specify the directory where the mscgen tool resides. If left empty the tool is assumed to +# be found in the default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a call dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable call graphs for selected +# functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will +# generate a caller dependency graph for every global function or class method. +# Note that enabling this option will significantly increase the time of a run. +# So in most cases it will be better to enable caller graphs for selected +# functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are png, jpg, or gif +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the number +# of direct children of the root node in a graph is already larger than +# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, which results in a white background. +# Warning: Depending on the platform used, enabling this option may lead to +# badly anti-aliased labels on the edges of a graph (i.e. they become hard to +# read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to the search engine +#--------------------------------------------------------------------------- + +# The SEARCHENGINE tag specifies whether or not a search engine should be +# used. If set to NO the values of all tags below this one will be ignored. + +SEARCHENGINE = NO Index: branches/eam_branches/20090522/ppNoiseMap/Makefile.am =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/Makefile.am (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/Makefile.am (revision 24557) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *~ core core.* Index: branches/eam_branches/20090522/ppNoiseMap/autogen.sh =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/autogen.sh (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/autogen.sh (revision 24557) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppImage +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOLIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: branches/eam_branches/20090522/ppNoiseMap/configure.ac =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/configure.ac (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/configure.ac (revision 24557) @@ -0,0 +1,35 @@ +AC_PREREQ(2.61) + +AC_INIT([ppNoiseMap], [0.0.1], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror" + +IPP_VERSION + +AC_SUBST([PPNOISEMAP_CFLAGS]) +AC_SUBST([PPNOISEMAP_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile + Doxyfile +]) +AC_OUTPUT Index: branches/eam_branches/20090522/ppNoiseMap/src/Makefile.am =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/Makefile.am (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/Makefile.am (revision 24557) @@ -0,0 +1,48 @@ +bin_PROGRAMS = ppNoiseMap + +noinst_HEADERS = \ + ppNoiseMap.h + +if HAVE_SVNVERSION +PPNOISEMAP_VERSION=`$(SVNVERSION) ..` +else +PPNOISEMAP_VERSION="UNKNOWN" +endif + +if HAVE_SVN +PPIMAGE_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'` +PPIMAGE_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'` +else +PPIMAGE_BRANCH="UNKNOWN" +PPIMAGE_SOURCE="UNKNOWN" +endif + +# Force recompilation of ppNoiseMapVersion.c, since it gets the version information +ppNoiseMapVersion.c: ppNoiseMapVersionDefinitions.h +ppNoiseMapVersionDefinitions.h: ppNoiseMapVersionDefinitions.h.in FORCE + -$(RM) ppNoiseMapVersionDefinitions.h + $(SED) -e "s|@PPNOISEMAP_VERSION@|\"$(PPNOISEMAP_VERSION)\"|" -e "s|@PPNOISEMAP_BRANCH@|\"$(PPNOISEMAP_BRANCH)\"|" -e "s|@PPNOISEMAP_SOURCE@|\"$(PPNOISEMAP_SOURCE)\"|" ppNoiseMapVersionDefinitions.h.in > ppNoiseMapVersionDefinitions.h +FORCE: ; + +BUILT_SOURCES = ppNoiseMapVersionDefinitions.h + +ppNoiseMap_CFLAGS = $(PPNOISEMAP_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) +ppNoiseMap_LDFLAGS = $(PPNOISEMAP_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS) +ppNoiseMap_SOURCES = \ + ppNoiseMap.c \ + ppNoiseMapArguments.c \ + ppNoiseMapParseCamera.c \ + ppNoiseMapLoop.c \ + ppNoiseMapCleanup.c \ + ppNoiseMapVersion.c \ + ppNoiseMapReadout.c \ + ppNoiseMapStats.c \ + ppNoiseMapDefineFile.c + +CLEANFILES = *~ + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMap.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMap.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMap.c (revision 24557) @@ -0,0 +1,42 @@ +#include "ppNoiseMap.h" + +int main(int argc, char **argv) { + + psLibInit(NULL); + + psTimerStart(TIMER_TOTAL); + + // Parse the configuration and arguments + // Open the input image + // Determine camera, format from header if not already defined + // Construct camera in preparation for reading + pmConfig *config = ppNoiseMapArguments(argc, argv); + if (config == NULL) { + psErrorStackPrint(stderr, "Unable to parse command-line arguments."); + ppNoiseMapCleanup(config); + exit(PS_EXIT_CONFIG_ERROR); + } + + ppNoiseMapVersionPrint(); + + // define the active I/O files + if (!ppNoiseMapParseCamera(config)) { + psErrorStackPrint(stderr, "Unable to parse camera."); + ppNoiseMapCleanup(config); + exit(PS_EXIT_CONFIG_ERROR); + } + + // Image Arithmetic Loop + if (!ppNoiseMapLoop(config)) { + psErrorStackPrint(stderr, "Unable to loop over input"); + ppNoiseMapCleanup(config); + exit(PS_EXIT_SYS_ERROR); + } + + psLogMsg("ppNoiseMap", PS_LOG_INFO, "Complete ppNoiseMap run: %f sec\n", psTimerMark(TIMER_TOTAL)); + + // Cleaning up + ppNoiseMapCleanup(config); + + return PS_EXIT_SUCCESS; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMap.h =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMap.h (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMap.h (revision 24557) @@ -0,0 +1,53 @@ +#ifndef PP_NOISE_MAP_H +#define PP_NOISE_MAP_H + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include // for strcasecmp +#include // for unlink +#include "pslib.h" +#include "psmodules.h" + +#define RECIPE_NAME "PPNOISEMAP" // Name of the recipe to use +#define TIMER_TOTAL "PPNOISEMAP.TOTAL" // Name of timer for total time + +/// Get the configuration +pmConfig *ppNoiseMapArguments(int argc, char **argv); + +/// Determine what type of camera, and initialise +bool ppNoiseMapParseCamera(pmConfig *config); + +/// Loop over the input +bool ppNoiseMapLoop(pmConfig *config); + +/// free memory, check for leaks +void ppNoiseMapCleanup (pmConfig *config); + +/// define the input files +bool ppNoiseMapDefineFile (pmConfig *config, pmFPA *input, char *filerule, char *argname, pmFPAfileType fileType); + +/// Return short version information +psString ppNoiseMapVersion(void); + +/// Return software source +psString ppNoiseMapSource(void); + +/// Return long version information +psString ppNoiseMapVersionLong(void); + +/// Populate the header with version information for all dependencies +bool ppNoiseMapVersionHeader(psMetadata *header); + +/// Print version information +void ppNoiseMapVersionPrint(void); + +/// perform the noise measurement +bool ppNoiseMapReadout (pmConfig *config, pmFPAview *view); + +/// measure the noise for the readout +bool ppNoiseMapStats(pmReadout *out, const pmReadout *in, psImageMaskType maskVal, int xBin, int yBin); +#endif Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapArguments.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapArguments.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapArguments.c (revision 24557) @@ -0,0 +1,39 @@ +#include "ppNoiseMap.h" + +static void usage (void) { + fprintf(stderr, "USAGE: ppNoiseMap [-file INPUT.fits] [-list INPUT.txt] OUTPUT\n\n"); + fprintf(stderr, "\n"); + exit (2); +} + +pmConfig *ppNoiseMapArguments(int argc, char **argv) +{ + if (argc == 1) { + usage(); + } + + if (psArgumentGet (argc, argv, "-version")) { + psString version; + version = ppNoiseMapVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = psModulesVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + version = psLibVersionLong(); fprintf (stdout, "%s\n", version); psFree (version); + exit (0); + } + + // load the site-wide configuration information + pmConfig *config = pmConfigRead(&argc, argv, RECIPE_NAME); + if (config == NULL) { + psErrorStackPrint(stderr, "Can't find site configuration!\n"); + exit(EXIT_FAILURE); + } + + // the input file is a required argument; if not found, we will exit + pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list"); + + if (argc != 2) usage (); + + // Add the input and output images (which remain on the command-line) to the arguments list + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]); + + return config; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapCleanup.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapCleanup.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapCleanup.c (revision 24557) @@ -0,0 +1,20 @@ +# include "ppNoiseMap.h" + +void ppNoiseMapCleanup (pmConfig *config) +{ + // Free memory used by ppNoiseMap + psFree(config); + + // Free memory used by psModules + pmSourceFitSetDone (); + pmConceptsDone(); + pmConfigDone(); + pmModelClassCleanup(); + + // Free memory used by psLib + psLibFinalize(); + + fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, stdout, false), "ppNoiseMap"); + + return; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapDefineFile.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapDefineFile.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapDefineFile.c (revision 24557) @@ -0,0 +1,34 @@ +# include "ppNoiseMap.h" + +bool ppNoiseMapDefineFile(pmConfig *config, pmFPA *input, char *filerule, char *argname, pmFPAfileType fileType) +{ + bool status; + pmFPAfile *file = NULL; // File to be defined + + if (!file) { + // look for the file on the RUN metadata + file = pmFPAfileDefineFromRun(&status, NULL, config, filerule); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to load file definition"); + return false; + } + } + if (!file) { + // look for the file on the argument list + file = pmFPAfileDefineFromArgs(&status, config, filerule, argname); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "failed to load file definition"); + return false; + } + } + + if (!file) { + return false; + } + + if (file->type != fileType) { + psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType)); + return false; + } + return true; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapLoop.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapLoop.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapLoop.c (revision 24557) @@ -0,0 +1,103 @@ +#include "ppNoiseMap.h" + +#define ESCAPE(MESSAGE) { \ + psError(PS_ERR_UNKNOWN, false, MESSAGE); \ + psFree(view); \ + return false; \ +} + +bool ppNoiseMapLoop(pmConfig *config) +{ + bool status; // Status of MD lookup + pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPNOISEMAP.INPUT"); + if (!status) { + psErrorStackPrint(stderr, "Can't find input data!\n"); + ppNoiseMapCleanup(config); + exit(PS_EXIT_PROG_ERROR); + } + + pmConfigCamerasCull(config, NULL); + pmConfigRecipesCull(config, "PPNOISEMAP,MASKS"); + + pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest + pmHDU *lastHDU = NULL; // Last HDU that was updated + + // files associated with the science image + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for FPA"); + } + + pmChip *chip; // Chip from FPA + while ((chip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { + psLogMsg ("ppNoiseMapLoop", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); + if (!chip->process || !chip->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for Chip"); + } + + // load all cells for this chip before generating output file fpa: + pmCell *cell; // Cell from chip + while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + psLogMsg ("ppNoiseMapLoop", 5, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (!cell->process || !cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for Cell"); + } + + // Put version information into the header + pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell); + if (hdu && hdu != lastHDU) { + ppNoiseMapVersionHeader(hdu->header); + lastHDU = hdu; + } + + // XXX for now, skip the video cells (cell->readouts->n > 1) + if (cell->readouts->n > 1) { + psWarning ("Skipping Video Cell for ppNoiseMap"); + continue; + } + + // process each of the readouts + pmReadout *readout; // Readout from cell + while ((readout = pmFPAviewNextReadout (view, input->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + ESCAPE("load failure for Readout"); + } + if (!readout->data_exists) { + continue; + } + } + } + + if (!ppNoiseMapReadout(config, view)) { + ESCAPE("Unable to measure noise map for readout"); + } + + // close the cells + while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + if (!cell->process || !cell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + ESCAPE("save failure for Cell"); + } + } + + // Close chip + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + ESCAPE("save failure for Chip"); + } + } + + // Output and Close FPA + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + ESCAPE("save failure for FPA"); + } + psFree(view); + + return true; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapParseCamera.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapParseCamera.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapParseCamera.c (revision 24557) @@ -0,0 +1,59 @@ +# include "ppNoiseMap.h" + +bool ppNoiseMapParseCamera(pmConfig *config) { + + bool status; + + if (!ppNoiseMapDefineFile(config, NULL, "PPNOISEMAP.INPUT", "INPUT", PM_FPA_FILE_IMAGE)) { + psError(PS_ERR_IO, false, "Can't find an input image source"); + return NULL; + } + pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPNOISEMAP.INPUT"); // Input file + psAssert(input, "We just put it there!"); + + // the recipe is only fully parsed after the camera is first determined + psMetadata *recipe = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME); + + int xBin1 = psMetadataLookupS32(&status, recipe, "XBIN"); + int yBin1 = psMetadataLookupS32(&status, recipe, "YBIN"); + + // this generates an output that maps to the same input pixels: + // pmFPAfile *outImage = pmFPAfileDefineOutput(config, input->fpa, "PPNOISEMAP.OUTPUT"); + + // this output results in new pixels with binning: + pmFPAfile *outImage = pmFPAfileDefineFromFPA(config, input->fpa, xBin1, yBin1, "PPNOISEMAP.OUTPUT"); + + if (!outImage) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPNOISEMAP.OUTPUT")); + return NULL; + } + if (outImage->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPNOISEMAP.OUTPUT is not of type IMAGE"); + return NULL; + } + outImage->save = true; + + // outImage is used as a carrier: input to chipImage -> require the data to remain at the CHIP level + outImage->freeLevel = PS_MIN(outImage->freeLevel, PM_FPA_LEVEL_CHIP); + outImage->dataLevel = outImage->freeLevel; + outImage->fileLevel = PS_MIN(outImage->fileLevel, outImage->dataLevel); + + // the input data is the same as the outImage data : force the free levels to match + input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel); + + if (psTraceGetLevel("ppNoiseMap.config") > 0) { + // Get a look inside all the files. + psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL); + psMetadataItem *item; // Item from iteration + fprintf(stderr, "Files:\n"); + while ((item = psMetadataGetAndIncrement(filesIter))) { + pmFPAfile *file = item->data.V; // File of interest + fprintf(stderr, "%s: %p %p %p (%p) %p\n", file->name, + file->src, file->fpa, + file->camera, file->fpa->camera, file->format); + } + psFree(filesIter); + } + + return true; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapReadout.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapReadout.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapReadout.c (revision 24557) @@ -0,0 +1,37 @@ +#include "ppNoiseMap.h" + +bool ppNoiseMapReadout (pmConfig *config, pmFPAview *view) { + + pmFPAfile *outFile = psMetadataLookupPtr(NULL, config->files, "PPNOISEMAP.OUTPUT"); + if (outFile == NULL) return false; + + pmChip *inChip = pmFPAviewThisChip (view, outFile->src); + pmChip *outChip = pmFPAviewThisChip (view, outFile->fpa); + if (!pmChipCopyStructure (outChip, inChip, outFile->xBin, outFile->yBin)) { + psError(PS_ERR_UNKNOWN, false, "Unable to copy chip structure."); + return false; + } + + pmCell *cell; + pmReadout *inReadout, *outReadout; + + while ((cell = pmFPAviewNextCell (view, outFile->src, 1)) != NULL) { + psLogMsg ("ppNoiseMapReadout", 5, "rebin: Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); + if (! cell->process || ! cell->file_exists) { continue; } + + // process each of the readouts + while ((inReadout = pmFPAviewNextReadout (view, outFile->src, 1)) != NULL) { + if (! inReadout->data_exists) { continue; } + + outReadout = pmFPAviewThisReadout (view, outFile->fpa); + + // run the rebin code + if (!ppNoiseMapStats(outReadout, inReadout, 0, outFile->xBin, outFile->yBin)) { + psError(PS_ERR_UNKNOWN, false, "Unable to rebin readout."); + return false; + } + } + } + + return true; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapStats.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapStats.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapStats.c (revision 24557) @@ -0,0 +1,99 @@ +#include "ppNoiseMap.h" + +bool ppNoiseMapStats(pmReadout *out, const pmReadout *in, psImageMaskType maskVal, int xBin, int yBin) +{ + PM_ASSERT_READOUT_NON_NULL(out, false); + PM_ASSERT_READOUT_NON_NULL(in, false); + + psImage *inImage = in->image, *inMask = in->mask; // Input image + int numColsIn = inImage->numCols, numRowsIn = inImage->numRows; // Size of input image + + psImageBinning *binning = psImageBinningAlloc(); // Binning instructions + binning->nXbin = xBin; + binning->nYbin = yBin; + binning->nXfine = numColsIn; + binning->nYfine = numRowsIn; + binning->nXskip = 0; + binning->nYskip = 0; + psImageBinningSetRuffSize(binning, PS_IMAGE_BINNING_CENTER); + + int numColsOut = binning->nXruff, numRowsOut = binning->nYruff; // Size of output image + + // re-use or re-generate the image + psImage *outImage; // Output image + if (out->image && out->image->numCols >= numColsOut && out->image->numRows >= numRowsOut) { + outImage = out->image; + } else { + outImage = out->image = psImageRecycle(out->image, numColsOut, numRowsOut, PS_TYPE_F32); + } + + psImage *outMask; // Output mask + if (out->mask && out->mask->numCols >= numColsOut && out->mask->numRows >= numRowsOut) { + outMask = out->mask; + } else { + outMask = out->mask = psImageRecycle(out->mask, numColsOut, numRowsOut, PS_TYPE_IMAGE_MASK); + } + + int nPixels = binning->nXbin * binning->nYbin; + psVector *values = psVectorAlloc (nPixels, PS_TYPE_F32); + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_STDEV); + + int xLast = numColsIn - 1, yLast = numRowsIn - 1; // Last index + int yStart = psImageBinningGetFineY(binning, 0); // Starting input y for binning + for (int yOut = 0; yOut < numRowsOut; yOut++) { + int yStop = psImageBinningGetFineY(binning, yOut + 1); // Stopping input y for binning + yStop = PS_MIN(yStop, yLast); + int xStart = psImageBinningGetFineX(binning, 0); // Starting input x for binning + for (int xOut = 0; xOut < numColsOut; xOut++) { + int xStop = psImageBinningGetFineX(binning, xOut + 1); // Stopping input x for binning + xStop = PS_MIN(xStop, xLast); + + // save the pixels for this subcell into the vector, + int numPix = 0; // Number of pixels + for (int y = yStart; y < yStop; y++) { + for (int x = xStart; x < xStop; x++) { + if (inMask && (inMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal)) { + continue; + } + values->data.F32[numPix] = inImage->data.F32[y][x]; + numPix++; + } + } + values->n = numPix; + + // measure the stats for this subcell + // Values to set + float imageValue; + psImageMaskType maskValue; + psStatsInit (stats); + if (!psVectorStats (stats, values, NULL, NULL, 0)) { + psWarning ("failure to measure stats for subcell %d,%d\n", xOut, yOut); + imageValue = NAN; + maskValue = maskVal; + } else { + imageValue = stats->robustStdev; + maskValue = 0; + } + outImage->data.F32[yOut][xOut] = imageValue; + outMask->data.PS_TYPE_IMAGE_MASK_DATA[yOut][xOut] = maskValue; + xStart = xStop; + } + yStart = yStop; + } + + psFree(binning); + psFree(values); + psFree(stats); + + out->data_exists = true; + if (out->parent) { + pmCell *outCell = out->parent; // Output cell + outCell->data_exists = outCell->parent->data_exists = true; + + // We would copy the concepts from the input cell, except that is done by pmFPACopy, + // pmChipCopyStructure, etc. This function just does the mechanics of binning. + // We don't even update the CELL.XBIN, CELL.YBIN because that would apply the correction twice. + } + + return true; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapVersion.c =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapVersion.c (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapVersion.c (revision 24557) @@ -0,0 +1,97 @@ +#include "ppNoiseMap.h" +#include "ppNoiseMapVersionDefinitions.h" + +#ifndef PPNOISEMAP_VERSION +#error "PPNOISEMAP_VERSION is not set" +#endif +#ifndef PPNOISEMAP_BRANCH +#error "PPNOISEMAP_BRANCH is not set" +#endif +#ifndef PPNOISEMAP_SOURCE +#error "PPNOISEMAP_SOURCE is not set" +#endif + +psString ppNoiseMapVersion(void) +{ + char *value = NULL; + psStringAppend(&value, "%s@%s", PPNOISEMAP_BRANCH, PPNOISEMAP_VERSION); + return value; +} + +psString ppNoiseMapSource(void) +{ + return psStringCopy(PPNOISEMAP_SOURCE); +} + +psString ppNoiseMapVersionLong(void) +{ + psString version = ppNoiseMapVersion(); // Version, to return + psString source = ppNoiseMapSource(); // Source + + psStringPrepend(&version, "ppNoiseMap "); + psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__); + psFree(source); + +#ifdef __OPTIMIZE__ + psStringAppend(&version, " optimised"); +#else + psStringAppend(&version, " unoptimised"); +#endif + + return version; +}; + +bool ppNoiseMapVersionHeader(psMetadata *header) +{ + PS_ASSERT_METADATA_NON_NULL(header, false); + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString history = NULL; // History string + psStringAppend(&history, "ppNoiseMap at %s", timeString); + psFree(timeString); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history); + psFree(history); + + psLibVersionHeader(header); + psModulesVersionHeader(header); + + psString version = ppNoiseMapVersion(); // ppNoiseMap software version + psString source = ppNoiseMapSource(); // ppNoiseMap software source + + psStringPrepend(&version, "ppNoiseMap version: "); + psStringPrepend(&source, "ppNoiseMap source: "); + + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version); + psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source); + + psFree(version); + psFree(source); + + return true; +} + + +void ppNoiseMapVersionPrint(void) +{ + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psLogMsg("ppNoiseMap", PS_LOG_INFO, "ppNoiseMap at %s", timeString); + psFree(timeString); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString ppNoiseMap = ppNoiseMapVersionLong(); // ppNoiseMap version + + psLogMsg("ppNoiseMap", PS_LOG_INFO, "%s", pslib); + psLogMsg("ppNoiseMap", PS_LOG_INFO, "%s", psmodules); + psLogMsg("ppNoiseMap", PS_LOG_INFO, "%s", ppNoiseMap); + + psFree(pslib); + psFree(psmodules); + psFree(ppNoiseMap); + + return; +} Index: branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapVersionDefinitions.h.in =================================================================== --- branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapVersionDefinitions.h.in (revision 24557) +++ branches/eam_branches/20090522/ppNoiseMap/src/ppNoiseMapVersionDefinitions.h.in (revision 24557) @@ -0,0 +1,8 @@ +#ifndef PPNOISEMAP_VERSION_DEFINITIONS_H +#define PPNOISEMAP_VERSION_DEFINITIONS_H + +#define PPNOISEMAP_VERSION @PPNOISEMAP_VERSION@ // SVN version +#define PPNOISEMAP_BRANCH @PPNOISEMAP_BRANCH@ // SVN branch +#define PPNOISEMAP_SOURCE @PPNOISEMAP_SOURCE@ // SVN source + +#endif Index: branches/eam_branches/20090522/ppSim/src/ppSimLoop.c =================================================================== --- branches/eam_branches/20090522/ppSim/src/ppSimLoop.c (revision 24529) +++ branches/eam_branches/20090522/ppSim/src/ppSimLoop.c (revision 24557) @@ -121,5 +121,5 @@ // build one here if (!readout->variance) { - if (!pmReadoutGenerateVariance(readout, true)) { + if (!pmReadoutGenerateVariance(readout, NULL, true)) { psError (PS_ERR_UNKNOWN, false, "trouble creating variance"); return false; Index: branches/eam_branches/20090522/ppSim/src/ppSimMergeReadouts.c =================================================================== --- branches/eam_branches/20090522/ppSim/src/ppSimMergeReadouts.c (revision 24529) +++ branches/eam_branches/20090522/ppSim/src/ppSimMergeReadouts.c (revision 24557) @@ -15,5 +15,5 @@ if (!inReadout->variance) { - if (!pmReadoutGenerateVariance(inReadout, true)) { + if (!pmReadoutGenerateVariance(inReadout, NULL, true)) { psError (PS_ERR_UNKNOWN, false, "trouble creating variance"); return false; Index: branches/eam_branches/20090522/ppSub/src/ppSubCamera.c =================================================================== --- branches/eam_branches/20090522/ppSub/src/ppSubCamera.c (revision 24529) +++ branches/eam_branches/20090522/ppSub/src/ppSubCamera.c (revision 24557) @@ -163,4 +163,25 @@ + // Now that the camera has been determined, we can read the recipe + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim + if (!recipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE); + return false; + } + if (psMetadataLookupBool(NULL, config->arguments, "-photometry")) { + psMetadataAddBool(recipe, PS_LIST_TAIL, "PHOTOMETRY", PS_META_REPLACE, + "Perform photometry?", true); + } + if (psMetadataLookupBool(NULL, config->arguments, "-inverse")) { + psMetadataAddBool(recipe, PS_LIST_TAIL, "INVERSE", PS_META_REPLACE, + "Generate inverse subtractions?", true); + } + + data->inverse = psMetadataLookupBool(NULL, recipe, "INVERSE"); + data->photometry = psMetadataLookupBool(NULL, recipe, "PHOTOMETRY"); + + bool mdok; // Status of MD lookup + bool saveConv = psMetadataLookupBool(&mdok, recipe, "SAVE.CONVOLVED"); // Save convolved images? + // Convolved input image pmFPAfile *inConvImage = defineOutputFile(config, input, true, "PPSUB.INPUT.CONV", PM_FPA_FILE_IMAGE); @@ -171,4 +192,8 @@ return false; } + if (saveConv) { + inConvImage->save = true; + inConvMask->save = true; + } if (inVar) { pmFPAfile *inConvVar = defineOutputFile(config, inConvImage, false, "PPSUB.INPUT.CONV.VARIANCE", @@ -177,4 +202,7 @@ psError(PS_ERR_UNKNOWN, false, "Unable to define output files"); return false; + } + if (saveConv) { + inConvVar->save = true; } } @@ -188,4 +216,8 @@ return false; } + if (saveConv) { + refConvImage->save = true; + refConvMask->save = true; + } if (refVar) { pmFPAfile *refConvVar = defineOutputFile(config, refConvImage, false, "PPSUB.REF.CONV.VARIANCE", @@ -195,25 +227,8 @@ return false; } - } - - - // Now that the camera has been determined, we can read the recipe - psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim - if (!recipe) { - psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE); - return false; - } - if (psMetadataLookupBool(NULL, config->arguments, "-photometry")) { - psMetadataAddBool(recipe, PS_LIST_TAIL, "PHOTOMETRY", PS_META_REPLACE, - "Perform photometry?", true); - } - if (psMetadataLookupBool(NULL, config->arguments, "-inverse")) { - psMetadataAddBool(recipe, PS_LIST_TAIL, "INVERSE", PS_META_REPLACE, - "Generate inverse subtractions?", true); - } - - data->inverse = psMetadataLookupBool(NULL, recipe, "INVERSE"); - data->photometry = psMetadataLookupBool(NULL, recipe, "PHOTOMETRY"); - + if (saveConv) { + refConvVar->save = true; + } + } // Output image Index: branches/eam_branches/20090522/ppSub/src/ppSubMatchPSFs.c =================================================================== --- branches/eam_branches/20090522/ppSub/src/ppSubMatchPSFs.c (revision 24529) +++ branches/eam_branches/20090522/ppSub/src/ppSubMatchPSFs.c (revision 24557) @@ -21,4 +21,21 @@ #include "ppSub.h" + +// Normalise a region on an image +static void normaliseRegion(psImage *image, // Image to normalise + const psRegion *region, // Region of image to normalise + float norm // Normalisation + ) +{ + if (!image) { + return; + } + psAssert(region, "Expect region"); + psImage *subImage = psImageSubset(image, *region); // Sub-image + psBinaryOp(subImage, subImage, "*", psScalarAlloc(norm, PS_TYPE_F32)); + psFree(subImage); + return; +} + bool ppSubMatchPSFs(ppSubData *data) @@ -166,4 +183,9 @@ ppSubDataQuality(data, error, PPSUB_FILES_ALL); return true; + } else if (error == PM_ERR_SMALL_AREA) { + psErrorStackPrint(stderr, "Insufficient area for PSF matching"); + psWarning("Insufficient area for PSF matching --- suspect bad data quality."); + ppSubDataQuality(data, error, PPSUB_FILES_ALL); + return true; } else { psError(PS_ERR_UNKNOWN, false, "Unable to match images."); @@ -172,4 +194,49 @@ } + // Need to be careful with the normalisation + // We will normalise everything to the normalisation of the *input* image + { + // Since the entries are MULTI, we have to retrieve them differently + psMetadataIterator *regIter = psMetadataIteratorAlloc(inConv->analysis, PS_LIST_HEAD, + "^" PM_SUBTRACTION_ANALYSIS_REGION "$"); + psMetadataIterator *modeIter = psMetadataIteratorAlloc(inConv->analysis, PS_LIST_HEAD, + "^" PM_SUBTRACTION_ANALYSIS_MODE "$"); + psMetadataIterator *normIter = psMetadataIteratorAlloc(inConv->analysis, PS_LIST_HEAD, + "^" PM_SUBTRACTION_ANALYSIS_NORM "$"); + psMetadataItem *regItem; // Item with region + while ((regItem = psMetadataGetAndIncrement(regIter))) { + psAssert(regItem->type == PS_DATA_REGION && regItem->data.V, "Expect region type"); + psRegion *region = regItem->data.V; // Region of interest + psMetadataItem *modeItem = psMetadataGetAndIncrement(modeIter); // Item with mode + psAssert(modeItem && modeItem->type == PS_TYPE_S32, "Expect subtraction mode"); + pmSubtractionMode mode = modeItem->data.S32; // Subtraction mode + psMetadataItem *normItem = psMetadataGetAndIncrement(normIter); // Item with normalisation + psAssert(normItem && normItem->type == PS_TYPE_F32 && isfinite(normItem->data.F32), + "Expect normalisation"); + float norm = normItem->data.F32; // Normalisation + + switch (mode) { + case PM_SUBTRACTION_MODE_1: // Convolved the input to match template + case PM_SUBTRACTION_MODE_DUAL: // Convolved both; template should have flux conserved + psLogMsg("ppSub", PS_LOG_INFO, "Correcting image for normalisation of %f\n", norm); + normaliseRegion(inConv->image, region, 1.0 / norm); + normaliseRegion(refConv->image, region, 1.0 / norm); + normaliseRegion(inConv->variance, region, 1.0 / PS_SQR(norm)); + normaliseRegion(refConv->variance, region, 1.0 / PS_SQR(norm)); + break; + case PM_SUBTRACTION_MODE_2: // Convolved the template to match input + // We're already happy! + psLogMsg("ppSub", PS_LOG_INFO, "Image normalisation is correct\n"); + break; + default: + psAbort("Invalid subtraction mode: %x", mode); + } + } + psFree(regIter); + psFree(modeIter); + psFree(normIter); + } + + pmConceptsCopyFPA(inConv->parent->parent->parent, inRO->parent->parent->parent, true, true); pmConceptsCopyFPA(refConv->parent->parent->parent, refRO->parent->parent->parent, true, true); Index: branches/eam_branches/20090522/ppSub/src/ppSubReadoutInverse.c =================================================================== --- branches/eam_branches/20090522/ppSub/src/ppSubReadoutInverse.c (revision 24529) +++ branches/eam_branches/20090522/ppSub/src/ppSubReadoutInverse.c (revision 24557) @@ -23,24 +23,13 @@ invRO->data_exists = invRO->parent->data_exists = invRO->parent->parent->data_exists = true; + // Get concepts from reference + pmFPAfile *refFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF.CONV"); // File with concepts + pmFPA *invFPA = invRO->parent->parent->parent; // Inverse FPA + pmConceptsCopyFPA(invFPA, refFile->fpa, true, true); + + // Get astrometry from (forward) subtraction pmChip *outChip = outRO->parent->parent; // Output chip pmFPA *outFPA = outChip->parent; // Output FPA - pmFPA *invFPA = invRO->parent->parent->parent; // Inverse FPA - - pmConceptsCopyFPA(invFPA, outFPA, true, true); - -#if 0 - // Copy astrometry over - pmHDU *outHDU = outFPA->hdu; // Output HDU pmChip *invChip = invRO->parent->parent; // Inverse chip - psFree(view); - if (!outHDU || !inHDU) { - psError(PS_ERR_UNKNOWN, false, "Unable to find HDU at FPA level to copy astrometry."); - return false; - } - if (!pmAstromReadWCS(invFPA, invChip, outHDU->header, 1.0)) { - psError(PS_ERR_UNKNOWN, false, "Unable to read WCS astrometry from PPSUB.OUTPUT."); - return false; - } -#endif pmHDU *invHDU = invFPA->hdu; // Inverse HDU if (!pmAstromWriteWCS(invHDU->header, outFPA, outChip, WCS_TOLERANCE)) { @@ -48,4 +37,9 @@ return false; } + // Read from newly written astrometry so that it exists in the "inverse" FPA (for sources) + if (!pmAstromReadWCS(invFPA, invChip, invHDU->header, 1.0)) { + psError(PS_ERR_UNKNOWN, false, "Unable to read WCS astrometry."); + return false; + } return true; Index: branches/eam_branches/20090522/ppSub/src/ppSubReadoutPhotometry.c =================================================================== --- branches/eam_branches/20090522/ppSub/src/ppSubReadoutPhotometry.c (revision 24529) +++ branches/eam_branches/20090522/ppSub/src/ppSubReadoutPhotometry.c (revision 24557) @@ -65,4 +65,7 @@ // equivalent to the minuend image. pmReadout *inRO = pmFPAfileThisReadout(config->files, view, name); // Readout with image and sources + if (psMetadataLookup(inRO->analysis, "PSPHOT.SOURCES")) { + psMetadataRemoveKey(inRO->analysis, "PSPHOT.SOURCES"); + } pmFPAfile *photFile = psMetadataLookupPtr(&mdok, config->files, "PSPHOT.INPUT"); // Photometry file Index: branches/eam_branches/20090522/psLib/configure.ac =================================================================== --- branches/eam_branches/20090522/psLib/configure.ac (revision 24529) +++ branches/eam_branches/20090522/psLib/configure.ac (revision 24557) @@ -392,4 +392,5 @@ test/sys/Makefile test/types/Makefile + test/optime/Makefile utils/Makefile ]) Index: branches/eam_branches/20090522/psLib/src/fits/psFits.c =================================================================== --- branches/eam_branches/20090522/psLib/src/fits/psFits.c (revision 24529) +++ branches/eam_branches/20090522/psLib/src/fits/psFits.c (revision 24557) @@ -931,9 +931,11 @@ } + // use strncmp so that we can resolve the string valuves that cfitsio puts into headers (RICE_1 GZIP_1) + // into psFitsCompressionType if (strcmp(string, "NONE") == 0) return PS_FITS_COMPRESS_NONE; - if (strcmp(string, "GZIP") == 0) return PS_FITS_COMPRESS_GZIP; - if (strcmp(string, "RICE") == 0) return PS_FITS_COMPRESS_RICE; - if (strcmp(string, "HCOMPRESS") == 0) return PS_FITS_COMPRESS_HCOMPRESS; - if (strcmp(string, "PLIO") == 0) return PS_FITS_COMPRESS_PLIO; + if (strncmp(string, "GZIP", 4) == 0) return PS_FITS_COMPRESS_GZIP; + if (strncmp(string, "RICE", 4) == 0) return PS_FITS_COMPRESS_RICE; + if (strncmp(string, "HCOMPRESS", 9) == 0) return PS_FITS_COMPRESS_HCOMPRESS; + if (strncmp(string, "PLIO", 4) == 0) return PS_FITS_COMPRESS_PLIO; psWarning("Unable to identify compression type (%s) --- none set.", string); Index: branches/eam_branches/20090522/psLib/src/fits/psFitsTable.c =================================================================== --- branches/eam_branches/20090522/psLib/src/fits/psFitsTable.c (revision 24529) +++ branches/eam_branches/20090522/psLib/src/fits/psFitsTable.c (revision 24557) @@ -390,9 +390,11 @@ -bool psFitsInsertTable(psFits* fits, - const psMetadata* header, - const psArray* table, - const char *extname, - bool after) +static bool fitsInsertTable(psFits* fits, // FITS file + const psMetadata* header, // FITS header to write + const psArray* table, // Table to write + const char *extname, // Extension name to give table + bool after, // Write table after current extension? + bool writeData // Write data? + ) { PS_ASSERT_FITS_NON_NULL(fits, false); @@ -403,5 +405,5 @@ long numRows = table->n; - if (numRows < 1) { + if (writeData && numRows < 1) { // no table data, what can I do? psError(PS_ERR_BAD_PARAMETER_SIZE, true, @@ -504,5 +506,5 @@ fits_create_tbl(fits->fd, BINARY_TBL, - table->n, // number of rows in table + writeData ? numRows : 0, // number of rows in table numColumns, // number of columns in table (char**)columnNames->data, // names of the columns @@ -524,5 +526,5 @@ // Insert the table fits_insert_btbl(fits->fd, - table->n, // number of rows in table + writeData ? numRows : 0, // number of rows in table numColumns, // number of columns in table (char**)columnNames->data, // names of the columns @@ -562,76 +564,78 @@ // cfitsio requires that we write the data by columns --- urgh! - psMetadataIteratorSet(colSpecsIter, PS_LIST_HEAD); - for (long colNum = 1; (colSpecItem = psMetadataGetAndIncrement(colSpecsIter)); colNum++) { - // Note: colNum is unit-indexed, because it's for cfitsio - colSpec *spec = colSpecItem->data.V; // The specification - if (PS_DATA_IS_PRIMITIVE(spec->type)) { - size_t dataSize = PSELEMTYPE_SIZEOF(spec->type); // Size (in bytes) of this type - psVector *columnData = psVectorAlloc(table->n, spec->type); // The raw row data, to be written - psVectorInit(columnData, 0); - for (long i = 0; i < table->n; i++) { - psMetadata *row = table->data[i]; // The row of interest - psMetadataItem *dataItem = psMetadataLookup(row, colSpecItem->name); // The value of interest - memcpy(&columnData->data.U8[i * dataSize], &dataItem->data, dataSize); + if (writeData) { + psMetadataIteratorSet(colSpecsIter, PS_LIST_HEAD); + for (long colNum = 1; (colSpecItem = psMetadataGetAndIncrement(colSpecsIter)); colNum++) { + // Note: colNum is unit-indexed, because it's for cfitsio + colSpec *spec = colSpecItem->data.V; // The specification + if (PS_DATA_IS_PRIMITIVE(spec->type)) { + size_t dataSize = PSELEMTYPE_SIZEOF(spec->type); // Size (in bytes) of this type + psVector *columnData = psVectorAlloc(table->n, spec->type); // The raw row data, to be written + psVectorInit(columnData, 0); + for (long i = 0; i < table->n; i++) { + psMetadata *row = table->data[i]; // The row of interest + psMetadataItem *dataItem = psMetadataLookup(row, colSpecItem->name); // Value of interest + memcpy(&columnData->data.U8[i * dataSize], &dataItem->data, dataSize); + } + + int fitsDataType; // Data type for cfitsio + p_psFitsTypeToCfitsio(spec->type, NULL, NULL, &fitsDataType); + fits_write_col(fits->fd, + fitsDataType, + colNum, // column number + 1, // first row + 1, // first element + table->n, // number of rows + columnData->data.U8, // the data + &status); + psFree(columnData); + } else { + switch (spec->type) { + case PS_DATA_STRING: { + psArray *strings = psArrayAlloc(table->n); // Array of strings + for (long i = 0; i < table->n; i++) { + psMetadata *row = table->data[i]; // The row of interest + strings->data[i] = psMemIncrRefCounter(psMetadataLookupStr(NULL, row, + colSpecItem->name)); + } + fits_write_col_str(fits->fd, colNum, 1, 1, table->n, (char**)strings->data, &status); + psFree(strings); + break; + } + case PS_DATA_VECTOR: { + size_t dataSize = PSELEMTYPE_SIZEOF(spec->vectorType); // Size of data, in bytes + psVector *columnData = psVectorAlloc(spec->size * table->n * dataSize, PS_TYPE_U8); + psVectorInit(columnData, 0); + for (long i = 0; i < table->n; i++) { + psMetadata *row = table->data[i]; // The row of interest + psMetadataItem* dataItem = psMetadataLookup(row, colSpecItem->name); + if (dataItem->type != PS_DATA_VECTOR) { + // Just in case --- get a zero instead of some weird result + continue; + } + psVector *vector = dataItem->data.V; + memcpy(&columnData->data.U8[i * dataSize * spec->size], vector->data.U8, + vector->n * dataSize); + } + + int fitsDataType; // Data type for cfitsio + p_psFitsTypeToCfitsio(spec->vectorType, NULL, NULL, &fitsDataType); + fits_write_col(fits->fd, fitsDataType, colNum, 1, 1, table->n * spec->size, + columnData->data.U8, &status); + psFree(columnData); + break; + } + default: + psAbort("Should never get here.\n"); + } } - int fitsDataType; // Data type for cfitsio - p_psFitsTypeToCfitsio(spec->type, NULL, NULL, &fitsDataType); - fits_write_col(fits->fd, - fitsDataType, - colNum, // column number - 1, // first row - 1, // first element - table->n, // number of rows - columnData->data.U8, // the data - &status); - psFree(columnData); - } else { - switch (spec->type) { - case PS_DATA_STRING: { - psArray *strings = psArrayAlloc(table->n); // Array of strings - for (long i = 0; i < table->n; i++) { - psMetadata *row = table->data[i]; // The row of interest - strings->data[i] = psMemIncrRefCounter(psMetadataLookupStr(NULL, row, - colSpecItem->name)); - } - fits_write_col_str(fits->fd, colNum, 1, 1, table->n, (char**)strings->data, &status); - psFree(strings); - break; - } - case PS_DATA_VECTOR: { - size_t dataSize = PSELEMTYPE_SIZEOF(spec->vectorType); // Size of data, in bytes - psVector *columnData = psVectorAlloc(spec->size * table->n * dataSize, PS_TYPE_U8); - psVectorInit(columnData, 0); - for (long i = 0; i < table->n; i++) { - psMetadata *row = table->data[i]; // The row of interest - psMetadataItem* dataItem = psMetadataLookup(row, colSpecItem->name); - if (dataItem->type != PS_DATA_VECTOR) { - // Just in case --- get a zero instead of some weird result - continue; - } - psVector *vector = dataItem->data.V; - memcpy(&columnData->data.U8[i * dataSize * spec->size], vector->data.U8, - vector->n * dataSize); - } - - int fitsDataType; // Data type for cfitsio - p_psFitsTypeToCfitsio(spec->vectorType, NULL, NULL, &fitsDataType); - fits_write_col(fits->fd, fitsDataType, colNum, 1, 1, table->n * spec->size, - columnData->data.U8, &status); - psFree(columnData); - break; - } - default: - psAbort("Should never get here.\n"); + // Check error status from writing column + if (status != 0) { + psFitsError(status, true, "Unable to write column %ld of FITS table", colNum); + psFree(colSpecsIter); + psFree(colSpecs); + return false; } - } - - // Check error status from writing column - if (status != 0) { - psFitsError(status, true, "Unable to write column %ld of FITS table", colNum); - psFree(colSpecsIter); - psFree(colSpecs); - return false; } } @@ -650,4 +654,42 @@ return true; } + + +bool psFitsInsertTable(psFits* fits, const psMetadata* header, const psArray* table, const char *extname, + bool after) +{ + PS_ASSERT_FITS_NON_NULL(fits, false); + PS_ASSERT_FITS_WRITABLE(fits, false); + return fitsInsertTable(fits, header, table, extname, after, true); +} + +bool psFitsWriteTableEmpty(psFits *fits, const psMetadata *header, const psMetadata *columns, + const char *extname) +{ + PS_ASSERT_FITS_NON_NULL(fits, false); + PS_ASSERT_FITS_WRITABLE(fits, false); + if (!psFitsMoveLast(fits)) { + psError(PS_ERR_UNKNOWN, false, "Unable to move to last extension to write table"); + return false; + } + psArray *table = psArrayAlloc(1); // Dummy table carrying column definitions + table->data[0] = psMemIncrRefCounter((psPtr)columns); // Casting away const + bool status = fitsInsertTable(fits, header, table, extname, true, false); // Status of insertion + psFree(table); + return status; +} + +bool psFitsInsertTableEmpty(psFits *fits, const psMetadata *header, const psMetadata *columns, + const char *extname, bool after) +{ + PS_ASSERT_FITS_NON_NULL(fits, false); + PS_ASSERT_FITS_WRITABLE(fits, false); + psArray *table = psArrayAlloc(1); // Dummy table carrying column definitions + table->data[0] = psMemIncrRefCounter((psPtr)columns); // Casting away const + bool status = fitsInsertTable(fits, header, table, extname, after, false); // Status of insertion + psFree(table); + return status; +} + bool psFitsUpdateTable(psFits* fits, Index: branches/eam_branches/20090522/psLib/src/fits/psFitsTable.h =================================================================== --- branches/eam_branches/20090522/psLib/src/fits/psFitsTable.h (revision 24529) +++ branches/eam_branches/20090522/psLib/src/fits/psFitsTable.h (revision 24557) @@ -90,4 +90,12 @@ ); +/// Write an empty table +bool psFitsWriteTableEmpty( + psFits *fits, ///< FITS file pointer + const psMetadata *header, ///< Header to write + const psMetadata *columns, ///< Column definitions; no data used except name,type + const char *extname ///< Extension name for table + ); + /** Inserts a whole FITS table. A new HDU of the type BINTABLE is inserted either * before or after, depending on the AFTER parameter, the current HDU. @@ -104,4 +112,14 @@ bool after ///< TRUE if insert is done after CHDU, otherwise table is inserted before CHDU ); + +/// Insert an empty table +bool psFitsInsertTableEmpty( + psFits *fits, ///< FITS file pointer + const psMetadata *header, ///< Header to write + const psMetadata *columns, ///< Column definitions; no data used except name,type + const char *extname, ///< Extension name for table + bool after ///< Insert after current HDU? + ); + /** Updates a FITS table. The current HDU type must be either Index: branches/eam_branches/20090522/psLib/src/imageops/psImageBackground.c =================================================================== --- branches/eam_branches/20090522/psLib/src/imageops/psImageBackground.c (revision 24529) +++ branches/eam_branches/20090522/psLib/src/imageops/psImageBackground.c (revision 24557) @@ -52,24 +52,41 @@ // Minimum and maximum values - float min = values->data.F32[0]; - float max = values->data.F32[0]; + float min = +PS_MAX_F32; + float max = -PS_MAX_F32; // select a subset of the image pixels to measure the stats long n = 0; // Number of actual pixels in subset - for (long i = 0; i < Nsubset; i++) { - double frnd = psRandomUniform(rng); - int pixel = Npixels * frnd; - int ix = pixel % nx; - int iy = pixel / nx; + if (Nsubset >= Npixels) { + // if we have an image smaller than Nsubset, just loop over the image pixels + for (int iy = 0; iy < ny; iy++) { + for (int ix = 0; ix < nx; ix++) { + if (!isfinite(image->data.F32[iy][ix]) || (mask && mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskValue)) { + continue; + } - if (!isfinite(image->data.F32[iy][ix]) || (mask && mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskValue)) { - continue; - } + float value = image->data.F32[iy][ix]; + min = PS_MIN(value, min); + max = PS_MAX(value, max); + values->data.F32[n] = value; + n++; + } + } + } else { + for (long i = 0; i < Nsubset; i++) { + double frnd = psRandomUniform(rng); + int pixel = Npixels * frnd; + int ix = pixel % nx; + int iy = pixel / nx; - float value = image->data.F32[iy][ix]; - min = PS_MIN(value, min); - max = PS_MIN(value, max); - values->data.F32[n] = value; - n++; + if (!isfinite(image->data.F32[iy][ix]) || (mask && mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] & maskValue)) { + continue; + } + + float value = image->data.F32[iy][ix]; + min = PS_MIN(value, min); + max = PS_MAX(value, max); + values->data.F32[n] = value; + n++; + } } if (n < 0.01*Nsubset) { Index: branches/eam_branches/20090522/psLib/src/imageops/psImageConvolve.c =================================================================== --- branches/eam_branches/20090522/psLib/src/imageops/psImageConvolve.c (revision 24529) +++ branches/eam_branches/20090522/psLib/src/imageops/psImageConvolve.c (revision 24557) @@ -122,4 +122,22 @@ return kernel; +} + +psKernel *psKernelCopy(const psKernel *in) +{ + PS_ASSERT_KERNEL_NON_NULL(in, NULL); + + psKernel *out = psAlloc(sizeof(psKernel)); // The copied kernel, to be returned + psMemSetDeallocator(out,(psFreeFunc)kernelFree); + + out->image = psImageCopy(NULL, in->image, PS_TYPE_KERNEL); + out->xMin = in->xMin; + out->xMax = in->xMax; + out->yMin = in->yMin; + out->yMax = in->yMax; + + kernelRedirects(out, out->image->numRows); + + return out; } Index: branches/eam_branches/20090522/psLib/src/imageops/psImageConvolve.h =================================================================== --- branches/eam_branches/20090522/psLib/src/imageops/psImageConvolve.h (revision 24529) +++ branches/eam_branches/20090522/psLib/src/imageops/psImageConvolve.h (revision 24557) @@ -96,4 +96,11 @@ ); +/// Copy a kernel +/// +/// Performs a deep copy of the input kernel +psKernel *psKernelCopy( + const psKernel *in ///< Kernel to be copied + ); + /// Checks the type of a particular pointer. /// Index: branches/eam_branches/20090522/psLib/src/imageops/psImageUnbin.c =================================================================== --- branches/eam_branches/20090522/psLib/src/imageops/psImageUnbin.c (revision 24529) +++ branches/eam_branches/20090522/psLib/src/imageops/psImageUnbin.c (revision 24557) @@ -30,5 +30,5 @@ // DX, DY are the binning factor // dx, dy is the distance is high-res pixels to the 0,0 corner of the first -// binned pixel. +// binned pixel. // XXX check that this is still consistent with psphotImageMedian... psImage *psImageUnbin(psImage *out, const psImage *in, const psImageBinning *binning) @@ -69,5 +69,5 @@ // (Xs,Ys) : (Xe,Ye) : binned pixel centers in unbinned coords // corresponding to (Ix,Iy), (Ix+1,Iy+1) - // XXX should this be "+ dx" and + dy? + // XXX should this be "+ dx" and + dy? int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, Ix + 0.5))); int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, Iy + 0.5))); @@ -76,28 +76,28 @@ for (int iy = Ys; (iy < Ye) && (iy < Ny); iy++) { - float dY = (iy - Ys) / (float) DY; - float rY = 1.0 - dY; + float dY = (iy - Ys) / (float) DY; + float rY = 1.0 - dY; float Vxs = V10*dY + V00*rY; float Vxe = V11*dY + V01*rY; - // Vxs = (V10 - V00)*(iy - Ys) / DY + V00; - // Vxe = (V11 - V01)*(iy - Ys) / DY + V01; - - // dVx = Vxs_1 - Vxs_1 - // dVx = (V10*dY_1 + V00*rY_1) - (V10*dY_0 + V00*rY_0); - // dY_0 = (iy - Ys)/DY = iy/DY - Ys/DY; - // dY_1 = (iy + 1 - Ys)/DY = iy/DY - Ys/DY + 1/DY; - // rY_0 = 1 - dY_0; - // rY_1 = 1 - dY_1; - // dVx = V10*(ddY) + V00*(drY); - // ddY = 1/DY; - // drY = -1/DY; - - // dVxs = (V10 - V00)/DY; - // dVxe = (V11 - V01)/DY; - - // ddV = (Vxe_1 - Vxs_1)/DX - (Vxe_0 - Vxs_0)/DX; - // ddV = (Vxe_1 - Vxe_0)/DX - (Vxs_1 - Vxs_0)/DX; - // ddV = (V11 - V01 - V10 + V00)/(DX*DY); + // Vxs = (V10 - V00)*(iy - Ys) / DY + V00; + // Vxe = (V11 - V01)*(iy - Ys) / DY + V01; + + // dVx = Vxs_1 - Vxs_1 + // dVx = (V10*dY_1 + V00*rY_1) - (V10*dY_0 + V00*rY_0); + // dY_0 = (iy - Ys)/DY = iy/DY - Ys/DY; + // dY_1 = (iy + 1 - Ys)/DY = iy/DY - Ys/DY + 1/DY; + // rY_0 = 1 - dY_0; + // rY_1 = 1 - dY_1; + // dVx = V10*(ddY) + V00*(drY); + // ddY = 1/DY; + // drY = -1/DY; + + // dVxs = (V10 - V00)/DY; + // dVxe = (V11 - V01)/DY; + + // ddV = (Vxe_1 - Vxs_1)/DX - (Vxe_0 - Vxs_0)/DX; + // ddV = (Vxe_1 - Vxe_0)/DX - (Vxs_1 - Vxs_0)/DX; + // ddV = (V11 - V01 - V10 + V00)/(DX*DY); float dV = (Vxe - Vxs) / DX; @@ -128,5 +128,5 @@ for (int ix = 0; ix < Xs; ix++) { vOut[iy][ix] = V; - // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); } V += dV; @@ -141,5 +141,5 @@ for (int ix = Xe; ix < Nx; ix++) { vOut[iy][ix] = V; - // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); } V += dV; @@ -163,5 +163,5 @@ for (int ix = Xs; (ix < Xe) && (ix < Nx); ix++) { vOut[iy][ix] = V; - // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); V += dV; } @@ -176,5 +176,5 @@ for (int ix = Xs; (ix < Xe) && (ix < Nx); ix++) { vOut[iy][ix] = V; - // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(ix, iy, in, binning)) < UNBIN_TOL*fabs(V)); V += dV; } @@ -186,13 +186,13 @@ { float V; - // center of last pixel - int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, 0 + 0.5))); - int Xe = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, nx - 0.5))); - int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, 0 + 0.5))); - int Ye = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, ny - 0.5))); + // center of last pixel + int Xs = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, 0 + 0.5))); + int Xe = PS_MAX (0, PS_MIN (Nx, psImageBinningGetFineX(binning, nx - 0.5))); + int Ys = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, 0 + 0.5))); + int Ye = PS_MAX (0, PS_MIN (Ny, psImageBinningGetFineY(binning, ny - 0.5))); // 0,0 V = vIn[0][0]; - // assert (fabs(V - psImageUnbinPixel(0, 0, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(0, 0, in, binning)) < UNBIN_TOL*fabs(V)); for (int iy = 0; iy < Ys; iy++) @@ -204,5 +204,5 @@ // Nx,0 V = vIn[0][nx-1]; - // assert (fabs(V - psImageUnbinPixel(Nx-1, 0, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(Nx-1, 0, in, binning)) < UNBIN_TOL*fabs(V)); for (int iy = 0; iy < Ys; iy++) @@ -214,5 +214,5 @@ // 0,Ny V = vIn[ny-1][0]; - // assert (fabs(V - psImageUnbinPixel(0, Ny-1, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(0, Ny-1, in, binning)) < UNBIN_TOL*fabs(V)); for (int iy = Ye; iy < Ny; iy++) @@ -224,5 +224,5 @@ // Nx,Ny V = vIn[ny-1][nx-1]; - // assert (fabs(V - psImageUnbinPixel(Nx-1, Ny-1, in, binning)) < UNBIN_TOL*fabs(V)); + // assert (fabs(V - psImageUnbinPixel(Nx-1, Ny-1, in, binning)) < UNBIN_TOL*fabs(V)); for (int iy = Ye; iy < Ny; iy++) @@ -243,6 +243,6 @@ double psImageUnbinPixel(const double xFine, const double yFine, // desired Unbinned point (parent coords) - const psImage *in, // binned image - const psImageBinning *binning) //!< Overhang + const psImage *in, // binned image + const psImageBinning *binning) //!< Overhang { PS_ASSERT_IMAGE_NON_NULL(in, NAN); @@ -273,5 +273,5 @@ if ((x < -nXedge) || (x > in->numCols + nXedge) || (y < -nYedge) || (y > in->numRows + nYedge)) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Point (%f,%f) lies outside binned image", x, y); + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Point (%lf,%lf) lies outside binned image", x, y); return NAN; } @@ -281,6 +281,6 @@ // if we have a single pixel, there is no spatial information if ((in->numCols == 1) && (in->numRows == 1)) { - const double value = in->data.F32[0][0]; - return value; + const double value = in->data.F32[0][0]; + return value; } @@ -306,21 +306,21 @@ // if Nx == 1, we have no x-dir spatial information if (in->numCols == 1) { - double V0 = in->data.F32[Ys][Xs]; - double V1 = in->data.F32[Ye][Xs]; - - const double value = V0*ry + V1*dy; - return value; - } + double V0 = in->data.F32[Ys][Xs]; + double V1 = in->data.F32[Ye][Xs]; + + const double value = V0*ry + V1*dy; + return value; + } // if Ny == 1, we have no y-dir spatial information if (in->numRows == 1) { - double V0 = in->data.F32[Ys][Xs]; - double V1 = in->data.F32[Ys][Xe]; - - const double value = V0*rx + V1*dx; - return value; - } - - // Vxy + double V0 = in->data.F32[Ys][Xs]; + double V1 = in->data.F32[Ys][Xe]; + + const double value = V0*rx + V1*dx; + return value; + } + + // Vxy double V00 = in->data.F32[Ys][Xs]; double V01 = in->data.F32[Ye][Xs]; @@ -332,33 +332,33 @@ // corners if ((dx < 0.0) && (dy < 0.0)) { - return V00; - } + return V00; + } if ((dx > 1.0) && (dy < 0.0)) { - return V10; - } + return V10; + } if ((dx < 0.0) && (dy > 1.0)) { - return V01; - } + return V01; + } if ((dx > 1.0) && (dy > 1.0)) { - return V11; - } + return V11; + } // sides if (dx < 0.0) { - value = V00*ry + V01*dy; - return value; - } + value = V00*ry + V01*dy; + return value; + } if (dy < 0.0) { - value = V00*rx + V10*dx; - return value; - } + value = V00*rx + V10*dx; + return value; + } if (dx > 1.0) { - value = V10*ry + V11*dy; - return value; - } + value = V10*ry + V11*dy; + return value; + } if (dy > 1.0) { - value = V01*rx + V11*dx; - return value; - } + value = V01*rx + V11*dx; + return value; + } // bilinear interpolation Index: branches/eam_branches/20090522/psLib/src/sys/psAbort.c =================================================================== --- branches/eam_branches/20090522/psLib/src/sys/psAbort.c (revision 24529) +++ branches/eam_branches/20090522/psLib/src/sys/psAbort.c (revision 24557) @@ -10,5 +10,5 @@ * @author Eric Van Alst, MHPCC * @author Joshua Hoblitt, University of Hawaii - * + * * @version $Revision: 1.16 $ $Name: not supported by cvs2svn $ * @date $Date: 2008-04-13 08:18:27 $ @@ -34,5 +34,5 @@ ...) { - psErrorStackPrint(stderr, "Aborting. Error stack:"); + psErrorStackPrint(stderr, "Aborting in function %s at %s:%d. Error stack:", func, file, lineno); va_list argPtr; // variable list arguement pointer @@ -51,12 +51,12 @@ void p_psAssert(const char *file, - unsigned int lineno, - const char *func, - const bool value, - const char *format, - ...) + unsigned int lineno, + const char *func, + const bool value, + const char *format, + ...) { if (value) return; - psErrorStackPrint(stderr, "Aborting. Error stack:"); + psErrorStackPrint(stderr, "Assertion failed in function %s at %s:%d. Error stack:", func, file, lineno); va_list argPtr; // variable list arguement pointer Index: branches/eam_branches/20090522/psLib/test/Makefile.am =================================================================== --- branches/eam_branches/20090522/psLib/test/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/psLib/test/Makefile.am (revision 24557) @@ -1,3 +1,3 @@ -SUBDIRS = tap pstap $(SRCDIRS) +SUBDIRS = tap pstap optime $(SRCDIRS) TESTS = test.pl Index: branches/eam_branches/20090522/psLib/test/astro/tap_psTime_01.c =================================================================== --- branches/eam_branches/20090522/psLib/test/astro/tap_psTime_01.c (revision 24529) +++ branches/eam_branches/20090522/psLib/test/astro/tap_psTime_01.c (revision 24557) @@ -842,10 +842,8 @@ { psMemId id = psMemGetId(); - bool status = psTimeConvert(NULL, PS_TIME_TAI); - - ok(status == false, "psTimeConvert(NULL, PS_TIME_TAI) returned NULL"); - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - - psFree(time2); + bool status= psTimeConvert(NULL, PS_TIME_TAI); + + ok(status== false, "psTimeConvert(NULL, PS_TIME_TAI) returned NULL"); + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } @@ -858,8 +856,7 @@ psMemId id = psMemGetId(); psTime *time1 = psTimeAlloc(PS_TIME_TAI); - psTime *time2 = psTimeConvert(time1,-100); - ok(time2 == NULL, "psTimeConvert(time1, -100) returned NULL"); - psFree(time1); - psFree(time2); + bool status = psTimeConvert(time1,-100); + ok(status == false, "psTimeConvert(time1, -100) returned false"); + psFree(time1); ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } @@ -873,8 +870,7 @@ psMemId id = psMemGetId(); psTime *time1 = psTimeAlloc(PS_TIME_UTC); - psTime *time2 = psTimeConvert(time1,-100); - ok(time2 == NULL, "psTimeConvert(time1, -100) returned NULL"); - psFree(time1); - psFree(time2); + bool status = psTimeConvert(time1,-100); + ok(status == false, "psTimeConvert(time1, -100) returned false"); + psFree(time1); ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } @@ -888,8 +884,7 @@ psMemId id = psMemGetId(); psTime *time1 = psTimeAlloc(PS_TIME_TT); - psTime *time2 = psTimeConvert(time1,-100); - ok(time2 == NULL, "psTimeConvert(time1, -100) returned NULL"); - psFree(time1); - psFree(time2); + bool status = psTimeConvert(time1,-100); + ok(status == false, "psTimeConvert(time1, -100) returned false"); + psFree(time1); ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } @@ -903,8 +898,7 @@ psTime *time1 = psTimeAlloc(PS_TIME_TAI); time1->type = -100; - psTime *time2 = psTimeConvert(time1,PS_TIME_TAI); - ok(time2 == NULL, "psTimeConvert(time1, PS_TIME_TAI) returned NULL"); - psFree(time1); - psFree(time2); + bool status = psTimeConvert(time1,PS_TIME_TAI); + ok(status == false, "psTimeConvert(time1, PS_TIME_TAI) returned false"); + psFree(time1); ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } @@ -919,8 +913,7 @@ psTime *time1 = psTimeAlloc(PS_TIME_TAI); time1->nsec = 2e9; - psTime *time2 = psTimeConvert(time1, PS_TIME_TAI); - ok(time2 == NULL, "psTimeConvert(time1, PS_TIME_TAI) returns NULL for incorrect psTime object"); - psFree(time1); - psFree(time2); + bool status = psTimeConvert(time1, PS_TIME_TAI); + ok(status == false, "psTimeConvert(time1, PS_TIME_TAI) returns NULL for incorrect psTime object"); + psFree(time1); ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } @@ -929,4 +922,5 @@ // psTimeConvert() //Attempt to convert a time to the same type + //Should return true because time->type == type { psMemId id = psMemGetId(); @@ -936,10 +930,6 @@ time1->type = PS_TIME_TAI; time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1, PS_TIME_TAI); - ok(time2 == time1, "psTimeConvert(time, ...) returns time for conversion to same type"); - is_long(time2->sec, 1, "time->sec"); - is_long(time2->nsec, 2, "time->nsec"); - ok(time2->type == PS_TIME_TAI, "time->type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1, PS_TIME_TAI); + ok(status == true, "psTimeConvert(time, ...) returns true for conversion to same type"); psFree(time1); @@ -957,10 +947,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_TAI); - ok(time2 == time1, "psTimeConvert(time, ...) returns time after conversion to a different type"); - is_long(time2->sec, testTime1SecondsTAI, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsTAI, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_TAI, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_TAI); + ok(status == true, "psTimeConvert(time, ...) returns true after conversion to a different type"); + is_long(time1->sec, testTime1SecondsTAI, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsTAI, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_TAI, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -978,10 +968,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_TT); - ok(time2 == time1, "psTimeConvert(time, ...) returns time after conversion to a different type"); - is_long(time2->sec, testTime1SecondsTT, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsTT, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_TT, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_TT); + ok(status == true, "psTimeConvert(time, ...) returns true after conversion to a different type"); + is_long(time1->sec, testTime1SecondsTT, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsTT, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_TT, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -999,10 +989,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_UT1); - ok(time2 == time1, "psTimeConvert(time, ...) returns time after conversion to a different type"); - is_long(time2->sec, testTime1SecondsUT1, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsUT1, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_UT1, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_UT1); + ok(status == true, "psTimeConvert(time, ...) returns true after conversion to a different type"); + is_long(time1->sec, testTime1SecondsUT1, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsUT1, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_UT1, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1019,10 +1009,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_UTC); - ok(time2 == time1, "psTimeConvert(time, ...) returns time after conversion to a different type"); - is_long(time2->sec, testTime1SecondsUTC, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsUTC, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_UTC, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_UTC); + ok(status == true, "psTimeConvert(time, ...) returns true after conversion to a different type"); + is_long(time1->sec, testTime1SecondsUTC, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsUTC, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_UTC, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1041,10 +1031,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_TT); - ok(time2 == time1, "psTimeConvert() returned time for conversion to same type"); - is_long(time2->sec, testTime1SecondsTT, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsTT, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_TT, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_TT); + ok(status == true, "psTimeConvert() returned true for conversion to same type"); + is_long(time1->sec, testTime1SecondsTT, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsTT, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_TT, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1062,10 +1052,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_UT1); - ok(time1 == time2, "psTimeConvert() returned time for conversion to same type"); - is_long(time2->sec, testTime1SecondsUT1, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsUT1, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_UT1, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_UT1); + ok(status == true, "psTimeConvert() returned true for conversion to same type"); + is_long(time1->sec, testTime1SecondsUT1, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsUT1, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_UT1, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1083,10 +1073,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_UTC); - ok(time2 == time1, "psTimeConvert() returned time for conversion to same type"); - is_long(time2->sec, testTime1SecondsUTC, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsUTC, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_UTC, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_UTC); + ok(status == true, "psTimeConvert() returned true for conversion to same type"); + is_long(time1->sec, testTime1SecondsUTC, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsUTC, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_UTC, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1104,10 +1094,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_TAI); - ok(time2 == time1, "psTimeConvert() returned time for conversion to same type"); - is_long(time2->sec, testTime1SecondsTAI, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsTAI, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_TAI, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_TAI); + ok(status == true, "psTimeConvert() returned true for conversion to same type"); + is_long(time1->sec, testTime1SecondsTAI, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsTAI, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_TAI, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1125,10 +1115,10 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1,PS_TIME_UT1); - ok(time1 == time2, "psTimeConvert() returned time for conversion to same type"); - is_long(time2->sec, testTime1SecondsUT1, "psTimeConvert() returned the correct ->sec"); - is_long(time2->nsec, testTime1NanosecondsUT1, "psTimeConvert() returned the correct ->nsec"); - ok(time2->type == PS_TIME_UT1, "psTimeConvert() returned the correct type"); - is_bool(time2->leapsecond, false, "time->leapsecond"); + bool status = psTimeConvert(time1,PS_TIME_UT1); + ok(status == true, "psTimeConvert() returned true for conversion to same type"); + is_long(time1->sec, testTime1SecondsUT1, "psTimeConvert() returned the correct ->sec"); + is_long(time1->nsec, testTime1NanosecondsUT1, "psTimeConvert() returned the correct ->nsec"); + ok(time1->type == PS_TIME_UT1, "psTimeConvert() returned the correct type"); + is_bool(time1->leapsecond, false, "time->leapsecond"); psFree(time1); @@ -1147,8 +1137,7 @@ time1->leapsecond = false; - psTime *time2 = psTimeConvert(time1, PS_TIME_UTC); - ok(time2 == NULL, "psTimeConvert() returned NULL for conversion from UT1"); - psFree(time1); - psFree(time2); + bool status = psTimeConvert(time1, PS_TIME_UTC); + ok(status == false, "psTimeConvert() returned false for conversion from UT1"); + psFree(time1); ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); } Index: branches/eam_branches/20090522/psLib/test/astro/tap_psTime_03.c =================================================================== --- branches/eam_branches/20090522/psLib/test/astro/tap_psTime_03.c (revision 24529) +++ branches/eam_branches/20090522/psLib/test/astro/tap_psTime_03.c (revision 24557) @@ -494,5 +494,5 @@ psFree(timeStr); - time = psTimeConvert(time, PS_TIME_TT); + bool status = psTimeConvert(time, PS_TIME_TT); timeStr = psTimeToISO(time); is_str(timeStr, testTimeBStrTT[i], "TT ISO string"); @@ -500,5 +500,5 @@ // Verify UTC ISO string - time = psTimeConvert(time, PS_TIME_UTC); + status = psTimeConvert(time, PS_TIME_UTC); time->leapsecond = testTimeBLeapsecond[i]; timeStr = psTimeToISO(time); @@ -506,5 +506,5 @@ psFree(timeStr); - time = psTimeConvert(time, PS_TIME_UT1); + status = psTimeConvert(time, PS_TIME_UT1); timeStr = psTimeToISO(time); is_str(timeStr, testTimeBStrUT1[i], "UT1 ISO string"); Index: branches/eam_branches/20090522/psLib/test/math/Makefile.am =================================================================== --- branches/eam_branches/20090522/psLib/test/math/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/psLib/test/math/Makefile.am (revision 24557) @@ -43,5 +43,4 @@ tap_psStats08 \ tap_psStats09 \ - tap_psStatsTiming \ tap_psFunc01 \ tap_psStats_Sample_01 \ Index: branches/eam_branches/20090522/psLib/test/math/tap_psStatsTiming.c =================================================================== --- branches/eam_branches/20090522/psLib/test/math/tap_psStatsTiming.c (revision 24529) +++ (revision ) @@ -1,828 +1,0 @@ -#include -#include -#include - -#include "tap.h" -#include "pstap.h" - -// example tap lines: -// ok(condition, "condition succeeded"); -// skip_start(condition, Nskip, "Skipping tests because of failure"); - -# define DTIME(A,B) ((A.tv_sec - B.tv_sec) + 1e-6*(A.tv_usec - B.tv_usec)) -struct timeval start, mark; - -int main (void) -{ - plan_tests(68); - -// diag("psStats timing tests"); - - // build a gauss-deviate vector (mean = 0.0, sigma = 1.0) for tests - psRandom *seed = psRandomAllocSpecific (PS_RANDOM_TAUS, 0); - psVector *rnd = psVectorAlloc (1000, PS_TYPE_F32); - for (int i = 0; i < rnd->n; i++) { - rnd->data.F32[i] = psRandomGaussian (seed); - } - -// diag ("timing for sample mean"); - /********** SAMPLE MEAN ***********/ - // test stat sample mean (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN); - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 0); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.1, "sample mean %f (mask: 0, range: 0): %.3f sec", stats->sampleMean, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample mean (mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN); - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.12, "sample mean %f (mask: 1, range: 0): %.3f sec", stats->sampleMean, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample mean (no mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 0); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.18, "sample mean %f (mask: 0, range: 1): %.3f sec", stats->sampleMean, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample mean (mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.2, "sample mean %f (mask: 1, range: 1): %.3f sec", stats->sampleMean, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample mean (mask, range : small sample) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - psVector *mask = psVectorAlloc (10, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[3] = 1; - int nOld = rnd->n; - - rnd->n = 10; - gettimeofday (&start, NULL); - for (int i = 0; i < 1000000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - rnd->n = nOld; - - psF64 delta = DTIME(mark, start); - ok (delta < 0.2, "sample mean %f (mask: 1, range: 1): %.3f sec", stats->sampleMean, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for sample median"); - /********** SAMPLE MEDIAN ***********/ - // test stat sample median (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN); - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 0); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 2.8, "sample median %f (mask: 0, range: 0): %.3f sec", stats->sampleMedian, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample median (mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN); - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 2.8, "sample median %f (mask: 1, range: 0): %.3f sec", stats->sampleMedian, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample median (no mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 0); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 2.8, "sample median %f (mask: 0, range: 1): %.3f sec", stats->sampleMedian, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample median (mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 2.8, "sample median %f (mask: 1, range: 1): %.3f sec", stats->sampleMedian, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for sample stdev"); - /********** SAMPLE STDEV ***********/ - // test stat sample stdev (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV); - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 0); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.2, "sample stdev %f (mask: 0, range: 0): %.3f sec", stats->sampleStdev, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample stdev (mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV); - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.27, "sample stdev %f (mask: 1, range: 0): %.3f sec", stats->sampleStdev, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample stdev (no mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 0); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.36, "sample stdev %f (mask: 0, range: 1): %.3f sec", stats->sampleStdev, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample stdev (mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.42, "sample stdev %f (mask: 1, range: 1): %.3f sec", stats->sampleStdev, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - // test stat sample stdev (mask, range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - psVector *mask = psVectorAlloc (10, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[1] = 1; - int nOld = rnd->n; - - rnd->n = 10; - gettimeofday (&start, NULL); - for (int i = 0; i < 1000000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - rnd->n = nOld; - - psF64 delta = DTIME(mark, start); - ok (delta < 0.42, "sample stdev %f (mask: 1, range: 1): %.3f sec", stats->sampleStdev, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for sample min,max"); - /*************** MIN,MAX ******************/ - // test stat min,max (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX); - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.17, "sample min,max %f,%f (mask: 0, range: 0): %.3f sec", stats->min, stats->max, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - // test stat min,max (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX); - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.18, "sample min,max %f,%f (mask: 1, range: 0): %.3f sec", stats->min, stats->max, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - // test stat min,max (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.22, "sample min,max %f,%f (mask: 0, range: 1): %.3f sec", stats->min, stats->max, delta); - psFree (stats); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - // test stat min,max (no mask, no range) - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX | PS_STAT_USE_RANGE); - stats->min = -10; - stats->max = +10; - psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); - psVectorInit (mask, 0); - mask->data.U8[100] = 1; - mask->data.U8[200] = 1; - mask->data.U8[300] = 1; - - gettimeofday (&start, NULL); - for (int i = 0; i < 10000; i++) - { - psVectorStats (stats, rnd, NULL, mask, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.26, "sample min,max %f,%f (mask: 1, range: 1): %.3f sec", stats->min, stats->max, delta); - psFree (stats); - psFree (mask); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for clipped stats"); - /********** CLIPPED STATS ***********/ - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); - psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.3, "clipped mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->clippedMean, stats->clippedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); - psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.5, "clipped mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->clippedMean, stats->clippedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); - psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 1.2, "clipped mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->clippedMean, stats->clippedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for robust stats"); - /********** ROBUST STATS ***********/ - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); - psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.3, "robust mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->robustMedian, stats->robustStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); - psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.5, "robust mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->robustMedian, stats->robustStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); - psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 1.2, "robust mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->robustMedian, stats->robustStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for fitted stats"); - /********** FITTED TIMING ***********/ - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); - psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.7, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); - psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.8, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); - psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 2.2, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("timing for fitted (v2) stats"); - /********** FITTED (v2) TIMING ***********/ - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); - psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.7, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); - psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 0.8, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); - psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); - for (int i = 0; i < rnd2->n; i++) - { - rnd2->data.F32[i] = psRandomGaussian (seed); - } - - gettimeofday (&start, NULL); - for (int i = 0; i < 1000; i++) - { - psVectorStats (stats, rnd2, NULL, NULL, 1); - } - gettimeofday (&mark, NULL); - psF64 delta = DTIME(mark, start); - ok (delta < 2.2, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); - psFree (stats); - psFree (rnd2); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("compare sample, robust, and fitted mean and stdev to theoretical"); - // compare SAMPLE, FITTED, ROBUST mean to theoretical - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); - psVector *sample = psVectorAlloc (1000, PS_TYPE_F32); - psVector *robust = psVectorAlloc (1000, PS_TYPE_F32); - psVector *fitted = psVectorAlloc (1000, PS_TYPE_F32); - - for (int i = 0; i < 1000; i++) - { - // generate a new sample - for (int j = 0; j < rnd->n; j++) { - rnd->data.F32[j] = psRandomGaussian (seed); - } - // measure the stats - psVectorStats (stats, rnd, NULL, NULL, 1); - sample->data.F32[i] = stats->sampleMean; - robust->data.F32[i] = stats->robustMedian; - fitted->data.F32[i] = stats->fittedMean; - } - psFree (stats); - - stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); - psVectorStats (stats, sample, NULL, NULL, 1); - ok (stats->sampleStdev < 2/sqrt(1000), "sample mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); - psVectorStats (stats, robust, NULL, NULL, 1); - ok (stats->sampleStdev < 2/sqrt(1000), "robust mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); - psVectorStats (stats, fitted, NULL, NULL, 1); - ok (stats->sampleStdev < 2/sqrt(1000), "fitted mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); - psFree (stats); - psFree (sample); - psFree (robust); - psFree (fitted); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - -// diag ("compare sample, robust, and fitted mean and stdev to theoretical"); - // compare SAMPLE, FITTED_V2, ROBUST mean to theoretical - { - psMemId id = psMemGetId(); - - psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); - psVector *sample = psVectorAlloc (1000, PS_TYPE_F32); - psVector *robust = psVectorAlloc (1000, PS_TYPE_F32); - psVector *fitted = psVectorAlloc (1000, PS_TYPE_F32); - - for (int i = 0; i < 1000; i++) - { - // generate a new sample - for (int j = 0; j < rnd->n; j++) { - rnd->data.F32[j] = psRandomGaussian (seed); - } - // measure the stats - psVectorStats (stats, rnd, NULL, NULL, 1); - sample->data.F32[i] = stats->sampleMean; - robust->data.F32[i] = stats->robustMedian; - fitted->data.F32[i] = stats->fittedMean; - } - psFree (stats); - - stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); - psVectorStats (stats, sample, NULL, NULL, 1); - ok (stats->sampleStdev < 2/sqrt(1000), "sample mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); - psVectorStats (stats, robust, NULL, NULL, 1); - ok (stats->sampleStdev < 2/sqrt(1000), "robust mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); - psVectorStats (stats, fitted, NULL, NULL, 1); - ok (stats->sampleStdev < 2/sqrt(1000), "fitted mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); - psFree (stats); - psFree (sample); - psFree (robust); - psFree (fitted); - - ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); - } - - return exit_status(); -} - Index: branches/eam_branches/20090522/psLib/test/math/tap_psStatsTiming.txt =================================================================== --- branches/eam_branches/20090522/psLib/test/math/tap_psStatsTiming.txt (revision 24529) +++ (revision ) @@ -1,47 +1,0 @@ - -Running tap_psStatsTiming on alala (dual AMD Opteron, 64bit, 2.2GHz) -yields the following timing results: - -# timing for sample mean (1000 loops of 10000 pts) -ok 1 - sample mean 0.009719 (mask: 0, range: 0): 0.072 sec -ok 3 - sample mean 0.011060 (mask: 1, range: 0): 0.119 sec -ok 5 - sample mean 0.009719 (mask: 0, range: 1): 0.170 sec -ok 7 - sample mean 0.011060 (mask: 1, range: 1): 0.198 sec - -# timing for sample median (1000 loops of 10000 pts) -ok 9 - sample median 0.021781 (mask: 0, range: 0): 2.625 sec -ok 11 - sample median 0.023795 (mask: 1, range: 0): 2.646 sec -ok 13 - sample median 0.021781 (mask: 0, range: 1): 2.703 sec -ok 15 - sample median 0.023795 (mask: 1, range: 1): 2.716 sec - -# timing for sample stdev (1000 loops of 10000 pts) -ok 17 - sample stdev 0.964753 (mask: 0, range: 0): 0.193 sec -ok 19 - sample stdev 0.965887 (mask: 1, range: 0): 0.257 sec -ok 21 - sample stdev 0.964753 (mask: 0, range: 1): 0.353 sec -ok 23 - sample stdev 0.965887 (mask: 1, range: 1): 0.401 sec - -# timing for sample min,max (1000 loops of 10000 pts) -ok 25 - sample min,max -3.205688,2.706797 (mask: 0, range: 0): 0.125 sec -ok 27 - sample min,max -3.205688,2.706797 (mask: 1, range: 0): 0.152 sec -ok 29 - sample min,max -3.205688,2.706797 (mask: 0, range: 1): 0.201 sec -ok 31 - sample min,max -3.205688,2.706797 (mask: 1, range: 1): 0.238 sec - -# timing for clipped stats -not ok 33 - clipped mean -0.047714, stdev 0.991979 (mask: 0, range: 0): 0.369 sec (1000 pts / 1000 loops) -not ok 35 - clipped mean 0.023963, stdev 0.972186 (mask: 0, range: 0): 1.219 sec (3000 pts / 1000 loops) -not ok 37 - clipped mean -0.007020, stdev 0.985410 (mask: 0, range: 0): 4.883 sec (10000 pts / 1000 loops) - -NOTE: these fail because they are being compared to the 'robust' stats -limits below. The clipped mean algorithm should not be so slow (and -apparently non-linear in npts). - -# timing for robust stats -ok 39 - robust mean 0.123348, stdev 1.014896 (mask: 0, range: 0): 0.187 sec (1000 pts / 1000 loops) -ok 41 - robust mean -0.006812, stdev 0.974468 (mask: 0, range: 0): 0.382 sec (3000 pts / 1000 loops) -ok 43 - robust mean -0.013591, stdev 1.001539 (mask: 0, range: 0): 1.076 sec (10000 pts / 1000 loops) - -# timing for fitted stats -ok 45 - fitted mean -0.029859, stdev 0.982947 (mask: 0, range: 0): 0.381 sec (1000 pts / 1000 loops) -ok 47 - fitted mean 0.014660, stdev 0.956168 (mask: 0, range: 0): 0.727 sec (3000 pts / 1000 loops) -ok 49 - fitted mean -0.008402, stdev 1.001366 (mask: 0, range: 0): 1.914 sec (10000 pts / 1000 loops) - Index: branches/eam_branches/20090522/psLib/test/optime/Makefile.am =================================================================== --- branches/eam_branches/20090522/psLib/test/optime/Makefile.am (revision 24557) +++ branches/eam_branches/20090522/psLib/test/optime/Makefile.am (revision 24557) @@ -0,0 +1,27 @@ +AM_CPPFLAGS = \ + $(SRCINC) \ + -I$(top_srcdir)/test/tap/src \ + -I$(top_srcdir)/test/pstap/src \ + $(PSLIB_CFLAGS) +AM_LDFLAGS = \ + $(top_builddir)/src/libpslib.la \ + $(top_builddir)/test/tap/src/libtap.la \ + $(top_builddir)/test/pstap/src/libpstap.la \ + $(PSLIB_LIBS) + +TEST_PROGS = \ + tap_psStatsTiming + +if BUILD_TESTS +bin_PROGRAMS = $(TEST_PROGS) +TESTS = $(TEST_PROGS) +else +check_PROGRAMS = $(TEST_PROGS) +endif + +CLEANFILES = $(tmp_files) core core.* *~ *.bb *.bbg *.da gmon.out + +tests: $(check_PROGRAMS) + $(top_srcdir)/test/test.pl + +test: check Index: branches/eam_branches/20090522/psLib/test/optime/tap_psStatsTiming.c =================================================================== --- branches/eam_branches/20090522/psLib/test/optime/tap_psStatsTiming.c (revision 24557) +++ branches/eam_branches/20090522/psLib/test/optime/tap_psStatsTiming.c (revision 24557) @@ -0,0 +1,828 @@ +#include +#include +#include + +#include "tap.h" +#include "pstap.h" + +// example tap lines: +// ok(condition, "condition succeeded"); +// skip_start(condition, Nskip, "Skipping tests because of failure"); + +# define DTIME(A,B) ((A.tv_sec - B.tv_sec) + 1e-6*(A.tv_usec - B.tv_usec)) +struct timeval start, mark; + +int main (void) +{ + plan_tests(68); + +// diag("psStats timing tests"); + + // build a gauss-deviate vector (mean = 0.0, sigma = 1.0) for tests + psRandom *seed = psRandomAllocSpecific (PS_RANDOM_TAUS, 0); + psVector *rnd = psVectorAlloc (1000, PS_TYPE_F32); + for (int i = 0; i < rnd->n; i++) { + rnd->data.F32[i] = psRandomGaussian (seed); + } + +// diag ("timing for sample mean"); + /********** SAMPLE MEAN ***********/ + // test stat sample mean (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN); + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 0); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.1, "sample mean %f (mask: 0, range: 0): %.3f sec", stats->sampleMean, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample mean (mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN); + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.12, "sample mean %f (mask: 1, range: 0): %.3f sec", stats->sampleMean, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample mean (no mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 0); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.18, "sample mean %f (mask: 0, range: 1): %.3f sec", stats->sampleMean, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample mean (mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.2, "sample mean %f (mask: 1, range: 1): %.3f sec", stats->sampleMean, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample mean (mask, range : small sample) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + psVector *mask = psVectorAlloc (10, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[3] = 1; + int nOld = rnd->n; + + rnd->n = 10; + gettimeofday (&start, NULL); + for (int i = 0; i < 1000000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + rnd->n = nOld; + + psF64 delta = DTIME(mark, start); + ok (delta < 0.2, "sample mean %f (mask: 1, range: 1): %.3f sec", stats->sampleMean, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for sample median"); + /********** SAMPLE MEDIAN ***********/ + // test stat sample median (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN); + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 0); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 2.8, "sample median %f (mask: 0, range: 0): %.3f sec", stats->sampleMedian, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample median (mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN); + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 2.8, "sample median %f (mask: 1, range: 0): %.3f sec", stats->sampleMedian, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample median (no mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 0); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 2.8, "sample median %f (mask: 0, range: 1): %.3f sec", stats->sampleMedian, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample median (mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 2.8, "sample median %f (mask: 1, range: 1): %.3f sec", stats->sampleMedian, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for sample stdev"); + /********** SAMPLE STDEV ***********/ + // test stat sample stdev (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV); + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 0); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.2, "sample stdev %f (mask: 0, range: 0): %.3f sec", stats->sampleStdev, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample stdev (mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV); + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.27, "sample stdev %f (mask: 1, range: 0): %.3f sec", stats->sampleStdev, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample stdev (no mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 0); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.36, "sample stdev %f (mask: 0, range: 1): %.3f sec", stats->sampleStdev, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample stdev (mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.42, "sample stdev %f (mask: 1, range: 1): %.3f sec", stats->sampleStdev, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + // test stat sample stdev (mask, range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_STDEV | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + psVector *mask = psVectorAlloc (10, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[1] = 1; + int nOld = rnd->n; + + rnd->n = 10; + gettimeofday (&start, NULL); + for (int i = 0; i < 1000000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + rnd->n = nOld; + + psF64 delta = DTIME(mark, start); + ok (delta < 0.42, "sample stdev %f (mask: 1, range: 1): %.3f sec", stats->sampleStdev, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for sample min,max"); + /*************** MIN,MAX ******************/ + // test stat min,max (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX); + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.17, "sample min,max %f,%f (mask: 0, range: 0): %.3f sec", stats->min, stats->max, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + // test stat min,max (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX); + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.18, "sample min,max %f,%f (mask: 1, range: 0): %.3f sec", stats->min, stats->max, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + // test stat min,max (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.22, "sample min,max %f,%f (mask: 0, range: 1): %.3f sec", stats->min, stats->max, delta); + psFree (stats); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + // test stat min,max (no mask, no range) + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX | PS_STAT_USE_RANGE); + stats->min = -10; + stats->max = +10; + psVector *mask = psVectorAlloc (1000, PS_TYPE_U8); + psVectorInit (mask, 0); + mask->data.U8[100] = 1; + mask->data.U8[200] = 1; + mask->data.U8[300] = 1; + + gettimeofday (&start, NULL); + for (int i = 0; i < 10000; i++) + { + psVectorStats (stats, rnd, NULL, mask, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.26, "sample min,max %f,%f (mask: 1, range: 1): %.3f sec", stats->min, stats->max, delta); + psFree (stats); + psFree (mask); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for clipped stats"); + /********** CLIPPED STATS ***********/ + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); + psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.3, "clipped mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->clippedMean, stats->clippedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); + psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.5, "clipped mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->clippedMean, stats->clippedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV); + psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 1.2, "clipped mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->clippedMean, stats->clippedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for robust stats"); + /********** ROBUST STATS ***********/ + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); + psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.3, "robust mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->robustMedian, stats->robustStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); + psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.5, "robust mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->robustMedian, stats->robustStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE); + psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 1.2, "robust mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->robustMedian, stats->robustStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for fitted stats"); + /********** FITTED TIMING ***********/ + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); + psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.7, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); + psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.8, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); + psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 2.2, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("timing for fitted (v2) stats"); + /********** FITTED (v2) TIMING ***********/ + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); + psVector *rnd2 = psVectorAlloc (1000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.7, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (1000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); + psVector *rnd2 = psVectorAlloc (3000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 0.8, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (3000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); + psVector *rnd2 = psVectorAlloc (10000, PS_TYPE_F32); + for (int i = 0; i < rnd2->n; i++) + { + rnd2->data.F32[i] = psRandomGaussian (seed); + } + + gettimeofday (&start, NULL); + for (int i = 0; i < 1000; i++) + { + psVectorStats (stats, rnd2, NULL, NULL, 1); + } + gettimeofday (&mark, NULL); + psF64 delta = DTIME(mark, start); + ok (delta < 2.2, "fitted mean %f, stdev %f (mask: 0, range: 0): %.3f sec (10000 pts / 1000 loops)", stats->fittedMean, stats->fittedStdev, delta); + psFree (stats); + psFree (rnd2); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("compare sample, robust, and fitted mean and stdev to theoretical"); + // compare SAMPLE, FITTED, ROBUST mean to theoretical + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV); + psVector *sample = psVectorAlloc (1000, PS_TYPE_F32); + psVector *robust = psVectorAlloc (1000, PS_TYPE_F32); + psVector *fitted = psVectorAlloc (1000, PS_TYPE_F32); + + for (int i = 0; i < 1000; i++) + { + // generate a new sample + for (int j = 0; j < rnd->n; j++) { + rnd->data.F32[j] = psRandomGaussian (seed); + } + // measure the stats + psVectorStats (stats, rnd, NULL, NULL, 1); + sample->data.F32[i] = stats->sampleMean; + robust->data.F32[i] = stats->robustMedian; + fitted->data.F32[i] = stats->fittedMean; + } + psFree (stats); + + stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); + psVectorStats (stats, sample, NULL, NULL, 1); + ok (stats->sampleStdev < 2/sqrt(1000), "sample mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); + psVectorStats (stats, robust, NULL, NULL, 1); + ok (stats->sampleStdev < 2/sqrt(1000), "robust mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); + psVectorStats (stats, fitted, NULL, NULL, 1); + ok (stats->sampleStdev < 2/sqrt(1000), "fitted mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); + psFree (stats); + psFree (sample); + psFree (robust); + psFree (fitted); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + +// diag ("compare sample, robust, and fitted mean and stdev to theoretical"); + // compare SAMPLE, FITTED_V2, ROBUST mean to theoretical + { + psMemId id = psMemGetId(); + + psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2); + psVector *sample = psVectorAlloc (1000, PS_TYPE_F32); + psVector *robust = psVectorAlloc (1000, PS_TYPE_F32); + psVector *fitted = psVectorAlloc (1000, PS_TYPE_F32); + + for (int i = 0; i < 1000; i++) + { + // generate a new sample + for (int j = 0; j < rnd->n; j++) { + rnd->data.F32[j] = psRandomGaussian (seed); + } + // measure the stats + psVectorStats (stats, rnd, NULL, NULL, 1); + sample->data.F32[i] = stats->sampleMean; + robust->data.F32[i] = stats->robustMedian; + fitted->data.F32[i] = stats->fittedMean; + } + psFree (stats); + + stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); + psVectorStats (stats, sample, NULL, NULL, 1); + ok (stats->sampleStdev < 2/sqrt(1000), "sample mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); + psVectorStats (stats, robust, NULL, NULL, 1); + ok (stats->sampleStdev < 2/sqrt(1000), "robust mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); + psVectorStats (stats, fitted, NULL, NULL, 1); + ok (stats->sampleStdev < 2/sqrt(1000), "fitted mean %f, stdev %f (1000 tries)", stats->sampleMean, stats->sampleStdev); + psFree (stats); + psFree (sample); + psFree (robust); + psFree (fitted); + + ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks"); + } + + return exit_status(); +} + Index: branches/eam_branches/20090522/psLib/test/optime/tap_psStatsTiming.txt =================================================================== --- branches/eam_branches/20090522/psLib/test/optime/tap_psStatsTiming.txt (revision 24557) +++ branches/eam_branches/20090522/psLib/test/optime/tap_psStatsTiming.txt (revision 24557) @@ -0,0 +1,47 @@ + +Running tap_psStatsTiming on alala (dual AMD Opteron, 64bit, 2.2GHz) +yields the following timing results: + +# timing for sample mean (1000 loops of 10000 pts) +ok 1 - sample mean 0.009719 (mask: 0, range: 0): 0.072 sec +ok 3 - sample mean 0.011060 (mask: 1, range: 0): 0.119 sec +ok 5 - sample mean 0.009719 (mask: 0, range: 1): 0.170 sec +ok 7 - sample mean 0.011060 (mask: 1, range: 1): 0.198 sec + +# timing for sample median (1000 loops of 10000 pts) +ok 9 - sample median 0.021781 (mask: 0, range: 0): 2.625 sec +ok 11 - sample median 0.023795 (mask: 1, range: 0): 2.646 sec +ok 13 - sample median 0.021781 (mask: 0, range: 1): 2.703 sec +ok 15 - sample median 0.023795 (mask: 1, range: 1): 2.716 sec + +# timing for sample stdev (1000 loops of 10000 pts) +ok 17 - sample stdev 0.964753 (mask: 0, range: 0): 0.193 sec +ok 19 - sample stdev 0.965887 (mask: 1, range: 0): 0.257 sec +ok 21 - sample stdev 0.964753 (mask: 0, range: 1): 0.353 sec +ok 23 - sample stdev 0.965887 (mask: 1, range: 1): 0.401 sec + +# timing for sample min,max (1000 loops of 10000 pts) +ok 25 - sample min,max -3.205688,2.706797 (mask: 0, range: 0): 0.125 sec +ok 27 - sample min,max -3.205688,2.706797 (mask: 1, range: 0): 0.152 sec +ok 29 - sample min,max -3.205688,2.706797 (mask: 0, range: 1): 0.201 sec +ok 31 - sample min,max -3.205688,2.706797 (mask: 1, range: 1): 0.238 sec + +# timing for clipped stats +not ok 33 - clipped mean -0.047714, stdev 0.991979 (mask: 0, range: 0): 0.369 sec (1000 pts / 1000 loops) +not ok 35 - clipped mean 0.023963, stdev 0.972186 (mask: 0, range: 0): 1.219 sec (3000 pts / 1000 loops) +not ok 37 - clipped mean -0.007020, stdev 0.985410 (mask: 0, range: 0): 4.883 sec (10000 pts / 1000 loops) + +NOTE: these fail because they are being compared to the 'robust' stats +limits below. The clipped mean algorithm should not be so slow (and +apparently non-linear in npts). + +# timing for robust stats +ok 39 - robust mean 0.123348, stdev 1.014896 (mask: 0, range: 0): 0.187 sec (1000 pts / 1000 loops) +ok 41 - robust mean -0.006812, stdev 0.974468 (mask: 0, range: 0): 0.382 sec (3000 pts / 1000 loops) +ok 43 - robust mean -0.013591, stdev 1.001539 (mask: 0, range: 0): 1.076 sec (10000 pts / 1000 loops) + +# timing for fitted stats +ok 45 - fitted mean -0.029859, stdev 0.982947 (mask: 0, range: 0): 0.381 sec (1000 pts / 1000 loops) +ok 47 - fitted mean 0.014660, stdev 0.956168 (mask: 0, range: 0): 0.727 sec (3000 pts / 1000 loops) +ok 49 - fitted mean -0.008402, stdev 1.001366 (mask: 0, range: 0): 1.914 sec (10000 pts / 1000 loops) + Index: branches/eam_branches/20090522/psModules/src/camera/pmFPABin.c =================================================================== --- branches/eam_branches/20090522/psModules/src/camera/pmFPABin.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/camera/pmFPABin.c (revision 24557) @@ -65,5 +65,7 @@ } - float imageValue, maskValue;// Values to set + // Values to set + float imageValue; + psImageMaskType maskValue; if (numPix > 0) { imageValue = sum / numPix; Index: branches/eam_branches/20090522/psModules/src/camera/pmFPACopy.c =================================================================== --- branches/eam_branches/20090522/psModules/src/camera/pmFPACopy.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/camera/pmFPACopy.c (revision 24557) @@ -70,4 +70,5 @@ psImageBinningSetRuffSize(binning, PS_IMAGE_BINNING_CENTER); *target = psImageAlloc(binning->nXruff, binning->nYruff, source->type.type); + psImageInit (*target, 0.0); return; } @@ -221,4 +222,11 @@ psTrace("psModules.camera", 3, "xFlip: %d; yFlip: %d\n", xFlip, yFlip); + // Blow away extant readouts + for (int i = 0; i < target->readouts->n; i++) { + psFree(target->readouts->data[i]); + target->readouts->data[i] = NULL; + } + target->readouts->n = 0; + // Perform deep copy of the images. I would prefer *not* to do a deep copy, in the interests of speed (we // still need to do another deep copy into the HDU for when we write out), but this is the only way I can @@ -242,4 +250,21 @@ readoutCopyComponent(&targetReadout->variance, sourceReadout->variance, binning, xFlip, yFlip, pixels); + // Copy covariance matrix: doesn't care about flips, etc. + if (sourceReadout->covariance) { + if (targetReadout->covariance) { + psFree(targetReadout->covariance); + } + targetReadout->covariance = psKernelCopy(sourceReadout->covariance); +#if 0 + if (binning) { + // XXX This isn't strictly correct, but we don't have a function that bins covariance matrices + // with unequal binning factors. + psKernel *covar = psImageCovarianceBin(PS_MAX(binning->nXbin, binning->nYbin), + targetReadout->covariance); + psFree(targetReadout->covariance); + targetReadout->covariance = covar; + } +#endif + } // Copy bias Index: branches/eam_branches/20090522/psModules/src/camera/pmFPAMaskWeight.c =================================================================== --- branches/eam_branches/20090522/psModules/src/camera/pmFPAMaskWeight.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/camera/pmFPAMaskWeight.c (revision 24557) @@ -199,7 +199,8 @@ } -bool pmReadoutSetVariance(pmReadout *readout, bool poisson) +bool pmReadoutSetVariance(pmReadout *readout, const psImage *noiseMap, bool poisson) { PS_ASSERT_PTR_NON_NULL(readout, false); + // check that the noiseMap (if it exists) matches the readout variance size) pmCell *cell = readout->parent; // The parent cell @@ -228,4 +229,5 @@ // a negative variance is non-sensical. if the image value drops below 1, the variance must be 1. + // XXX this calculation is wrong: limit is 1 e-, but this is in DN readout->variance = (psImage*)psUnaryOp(readout->variance, readout->variance, "abs"); readout->variance = (psImage*)psBinaryOp(readout->variance, readout->variance, "max", @@ -239,6 +241,12 @@ } - readout->variance = (psImage*)psBinaryOp(readout->variance, readout->variance, "+", - psScalarAlloc(readnoise*readnoise/gain/gain, PS_TYPE_F32)); + // apply a supplied readnoise map (NOTE: in DN, not electrons): + if (noiseMap) { + psImage *rdVar = (psImage*)psBinaryOp(NULL, (const psPtr) noiseMap, "*", (const psPtr) noiseMap); + readout->variance = (psImage*)psBinaryOp(readout->variance, readout->variance, "+", rdVar); + psFree (rdVar); + } else { + readout->variance = (psImage*)psBinaryOp(readout->variance, readout->variance, "+", psScalarAlloc(readnoise*readnoise/gain/gain, PS_TYPE_F32)); + } return true; @@ -247,5 +255,5 @@ // this function creates the variance pixels, or uses the existing variance pixels. it will set // the noise pixel values only if the variance image is not supplied -bool pmReadoutGenerateVariance(pmReadout *readout, bool poisson) +bool pmReadoutGenerateVariance(pmReadout *readout, const psImage *noiseMap, bool poisson) { PS_ASSERT_PTR_NON_NULL(readout, false); @@ -291,8 +299,8 @@ readout->variance = variance; - return pmReadoutSetVariance(readout, poisson); -} - -bool pmReadoutGenerateMaskVariance(pmReadout *readout, psImageMaskType satMask, psImageMaskType badMask, bool poisson) + return pmReadoutSetVariance(readout, noiseMap, poisson); +} + +bool pmReadoutGenerateMaskVariance(pmReadout *readout, psImageMaskType satMask, psImageMaskType badMask, const psImage *noiseMap, bool poisson) { PS_ASSERT_PTR_NON_NULL(readout, false); @@ -301,10 +309,10 @@ success &= pmReadoutGenerateMask(readout, satMask, badMask); - success &= pmReadoutGenerateVariance(readout, poisson); + success &= pmReadoutGenerateVariance(readout, noiseMap, poisson); return success; } -bool pmCellGenerateMaskVariance(pmCell *cell, psImageMaskType satMask, psImageMaskType badMask, bool poisson) +bool pmCellGenerateMaskVariance(pmCell *cell, psImageMaskType satMask, psImageMaskType badMask, const psImage *noiseMap, bool poisson) { PS_ASSERT_PTR_NON_NULL(cell, false); @@ -314,5 +322,5 @@ for (int i = 0; i < readouts->n; i++) { pmReadout *readout = readouts->data[i]; // The readout - success &= pmReadoutGenerateMaskVariance(readout, poisson, satMask, badMask); + success &= pmReadoutGenerateMaskVariance(readout, satMask, badMask, noiseMap, poisson); } Index: branches/eam_branches/20090522/psModules/src/camera/pmFPAMaskWeight.h =================================================================== --- branches/eam_branches/20090522/psModules/src/camera/pmFPAMaskWeight.h (revision 24529) +++ branches/eam_branches/20090522/psModules/src/camera/pmFPAMaskWeight.h (revision 24557) @@ -54,4 +54,5 @@ /// can't be generated. bool pmReadoutSetVariance(pmReadout *readout, ///< Readout for which to set variance + const psImage *noiseMap, ///< 2D image of the read noise in DN bool poisson ///< Include poisson variance (in addition to read noise)? ); @@ -72,4 +73,5 @@ /// with HDU entry). This is intended for most operations. bool pmReadoutGenerateVariance(pmReadout *readout, ///< Readout for which to generate variance + const psImage *noiseMap, ///< 2D image of the read noise in DN bool poisson ///< Include poisson variance (in addition to read noise)? ); @@ -81,4 +83,5 @@ psImageMaskType sat, ///< Mask value to give saturated pixels psImageMaskType bad, ///< Mask value to give bad (low) pixels + const psImage *noiseMap, ///< 2D image of the read noise in DN bool poisson ///< Include poisson variance (in addition to read noise)? ); @@ -90,4 +93,5 @@ psImageMaskType sat, ///< Mask value to give saturated pixels psImageMaskType bad, ///< Mask value to give bad (low) pixels + const psImage *noiseMap, ///< 2D image of the read noise in DN bool poisson ///< Include poisson variance (in addition to read noise)? ); Index: branches/eam_branches/20090522/psModules/src/camera/pmHDU.c =================================================================== --- branches/eam_branches/20090522/psModules/src/camera/pmHDU.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/camera/pmHDU.c (revision 24557) @@ -93,12 +93,31 @@ } + psTrace("psModules.camera", 5, "Reading the header...\n"); + + // The header may already exist (e.g., from doing concept writing at the PHU level) so we need to be + // careful. We read into a separate container and copy that over the top of anything that's already read. + psMetadata *header = psFitsReadHeader(NULL, fits); + if (!header) { + psError(PS_ERR_IO, false, "Unable to read header for extension %s\n", hdu->extname); + return false; + } + if (!hdu->header) { - psTrace("psModules.camera", 5, "Reading the header...\n"); - hdu->header = psFitsReadHeader(hdu->header, fits); - if (! hdu->header) { - psError(PS_ERR_IO, false, "Unable to read header for extension %s\n", hdu->extname); - return false; - } - } + hdu->header = header; + return true; + } + + psMetadataIterator *iter = psMetadataIteratorAlloc(header, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + const char *name = item->name; // Name of item + if (psMetadataLookup(hdu->header, name)) { + // It exists; clobber + psMetadataRemoveKey(hdu->header, name); + } + psMetadataAddItem(hdu->header, item, PS_LIST_TAIL, 0); + } + psFree(iter); + psFree(header); return true; Index: branches/eam_branches/20090522/psModules/src/concepts/pmConceptsStandard.c =================================================================== --- branches/eam_branches/20090522/psModules/src/concepts/pmConceptsStandard.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/concepts/pmConceptsStandard.c (revision 24557) @@ -36,9 +36,9 @@ // Format type for time typedef enum { - TIME_FORMAT_YYYYMMDD, // Date stored in YYYY-MM-DD order (ISO-standard) - TIME_FORMAT_DDMMYYYY, // Date stored in DD-MM-YYYY order - TIME_FORMAT_MMDDYYYY, // Date stored in MM-DD-YYYY order - TIME_FORMAT_JD, // Date stored as JD - TIME_FORMAT_MJD, // Date stored as MJD + TIME_FORMAT_YYYYMMDD, // Date stored in YYYY-MM-DD order (ISO-standard) + TIME_FORMAT_DDMMYYYY, // Date stored in DD-MM-YYYY order + TIME_FORMAT_MMDDYYYY, // Date stored in MM-DD-YYYY order + TIME_FORMAT_JD, // Date stored as JD + TIME_FORMAT_MJD, // Date stored as MJD } conceptTimeFormatType; @@ -391,4 +391,5 @@ } + // FPA.RA and FPA.DEC psMetadataItem *p_pmConceptFormat_FPA_Coords(const psMetadataItem *concept, @@ -410,9 +411,8 @@ // How to interpret the coordinates bool mdok = true; // Status of MD lookup - psMetadata *formats = psMetadataLookupMetadata(&mdok, - cameraFormat, - "FORMATS"); + psMetadata *formats = psMetadataLookupMetadata(&mdok, cameraFormat, "FORMATS"); + bool sexagesimal = false; // Write sexagesimal format? if (mdok && formats) { - psString format = psMetadataLookupStr(&mdok,formats, concept->name); + psString format = psMetadataLookupStr(&mdok, formats, concept->name); if (mdok && strlen(format) > 0) { if (strcasecmp(format, "HOURS") == 0) { @@ -428,26 +428,36 @@ coords /= defaultCoordScaling(concept); } + + psString ra = psMetadataLookupStr(&mdok, formats, "FPA.RA"); // Format for RA + psString dec = psMetadataLookupStr(&mdok, formats, "FPA.DEC"); // Format for Dec + if (ra && strcasecmp(ra, "HOURS") == 0 && dec && strcasecmp(dec, "DEGREES") == 0) { + sexagesimal = true; + } } else { coords /= defaultCoordScaling(concept); } - // We choose to write sexagesimal format - int big, medium; // Degrees and minutes - float small; // Seconds - bool negative = (coords < 0); // Are we working below zero? - coords = fabs(coords); - big = (int)abs(coords); - coords -= big; - medium = 60.0 * coords; - coords -= medium / 60.0; - small = 3600.0 * coords; - small = (float)((int)(small * 1000.0)) / 1000.0; - if (negative) { - big *= -1; - } - psString coordString = NULL; // String with the coordinates in sexagesimal format - psStringAppend(&coordString, "%d:%02d:%06.3f", big, medium, small); - psMetadataItem *coordItem = psMetadataItemAllocStr(concept->name, concept->comment, coordString); - psFree(coordString); + psMetadataItem *coordItem = NULL; // Item with coordinates, to return + if (sexagesimal) { + int big, medium; // Degrees and minutes + float small; // Seconds + bool negative = (coords < 0); // Are we working below zero? + coords = fabs(coords); + big = (int)abs(coords); + coords -= big; + medium = 60.0 * coords; + coords -= medium / 60.0; + small = 3600.0 * coords; + small = (float)((int)(small * 1000.0)) / 1000.0; + if (negative) { + big *= -1; + } + psString coordString = NULL; // String with the coordinates in sexagesimal format + psStringAppend(&coordString, "%d:%02d:%06.3f", big, medium, small); + coordItem = psMetadataItemAllocStr(concept->name, concept->comment, coordString); + psFree(coordString); + } else { + coordItem = psMetadataItemAllocF64(concept->name, concept->comment, coords); + } return coordItem; Index: branches/eam_branches/20090522/psModules/src/config/pmConfig.c =================================================================== --- branches/eam_branches/20090522/psModules/src/config/pmConfig.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/config/pmConfig.c (revision 24557) @@ -243,4 +243,6 @@ unsigned int numBadLines = 0; struct stat filestat; + + pmErrorRegister(); psTrace("psModules.config", 3, "Loading %s configuration from file %s\n", @@ -1496,4 +1498,5 @@ } } + globfree(&globList); } psFree (words); Index: branches/eam_branches/20090522/psModules/src/config/pmErrorCodes.dat =================================================================== --- branches/eam_branches/20090522/psModules/src/config/pmErrorCodes.dat (revision 24529) +++ branches/eam_branches/20090522/psModules/src/config/pmErrorCodes.dat (revision 24557) @@ -12,4 +12,5 @@ SKY Problem in sky STAMPS Unable to select stamps for PSF-matching +SMALL_AREA Small area for PSF-matching # these errors correspond to standard exit conditions ARGUMENTS Incorrect arguments Index: branches/eam_branches/20090522/psModules/src/detrend/pmDetrendDB.c =================================================================== --- branches/eam_branches/20090522/psModules/src/detrend/pmDetrendDB.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/detrend/pmDetrendDB.c (revision 24557) @@ -105,4 +105,5 @@ DETREND_STRING_CASE(FRINGE); DETREND_STRING_CASE(ASTROM); + DETREND_STRING_CASE(NOISEMAP); default: return NULL; Index: branches/eam_branches/20090522/psModules/src/detrend/pmDetrendDB.h =================================================================== --- branches/eam_branches/20090522/psModules/src/detrend/pmDetrendDB.h (revision 24529) +++ branches/eam_branches/20090522/psModules/src/detrend/pmDetrendDB.h (revision 24557) @@ -35,4 +35,5 @@ PM_DETREND_TYPE_BACKGROUND, PM_DETREND_TYPE_ASTROM, + PM_DETREND_TYPE_NOISEMAP, } pmDetrendType; Index: branches/eam_branches/20090522/psModules/src/detrend/pmDetrendThreads.c =================================================================== --- branches/eam_branches/20090522/psModules/src/detrend/pmDetrendThreads.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/detrend/pmDetrendThreads.c (revision 24557) @@ -60,4 +60,6 @@ } + // NOISEMAP : for now, not applied in the threaded loop + return true; } Index: branches/eam_branches/20090522/psModules/src/imcombine/pmSubtraction.c =================================================================== --- branches/eam_branches/20090522/psModules/src/imcombine/pmSubtraction.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/imcombine/pmSubtraction.c (revision 24557) @@ -196,49 +196,12 @@ { psKernel *convolved = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Convolved image - int numBytes = (2 * footprint + 1) * PSELEMTYPE_SIZEOF(PS_TYPE_F32); // Number of bytes to copy for (int y = -footprint; y <= footprint; y++) { - // Convolution with a delta function is just the value specified by the offset - memcpy(&convolved->kernel[y][-footprint], &image->kernel[y - v][-footprint - u], numBytes); + for (int x = -footprint; x <= footprint; x++) { + convolved->kernel[y][x] = image->kernel[y - v][x - u]; + } } return convolved; } -// Take a subset of an image -static inline psImage *convolveSubsetAlloc(psImage *image, // Image to be convolved - psRegion region, // Region of interest (with border) - bool threaded // Are we running threaded? - ) -{ - if (!image) { - return NULL; - } - // XXX if (threaded) { - // XXX psMutexLock(image); - // XXX } - psImage *subset = psImageSubset(image, region); // Subset image, to return - // XXX if (threaded) { - // XXX psMutexUnlock(image); - // XXX } - return subset; -} - -// Free the subset of an image -static inline void convolveSubsetFree(psImage *parent, // Parent image - psImage *child, // Child (subset) image - bool threaded // Are we running threaded? - ) -{ - if (!child || !parent) { - return; - } - // XXX if (threaded) { - // XXX psMutexLock(parent); - // XXX } - psFree(child); - // XXX if (threaded) { - // XXX psMutexUnlock(parent); - // XXX } - return; -} // Convolve an image using FFT @@ -256,13 +219,11 @@ region.y0 - size, region.y1 + size); // Add a border - bool threaded = pmSubtractionThreaded(); // Are we running threaded? - - psImage *subImage = convolveSubsetAlloc(image, border, threaded); // Subimage to convolve - psImage *subMask = convolveSubsetAlloc(mask, border, threaded); // Subimage mask + psImage *subImage = image ? psImageSubset(image, border) : NULL; // Subimage to convolve + psImage *subMask = mask ? psImageSubset(mask, border) : NULL; // Subimage mask psImage *convolved = psImageConvolveFFT(NULL, subImage, subMask, maskVal, kernel); // Convolution - convolveSubsetFree(image, subImage, threaded); - convolveSubsetFree(mask, subMask, threaded); + psFree(subImage); + psFree(subMask); // Now, we have to stick it in where it belongs @@ -300,9 +261,7 @@ region.y0 - size, region.y1 + size); // Add a border - bool threaded = pmSubtractionThreaded(); // Are we running threaded? - - psImage *subVariance = convolveSubsetAlloc(variance, border, threaded); // Variance map - psImage *subSys = convolveSubsetAlloc(sys, border, threaded); // Systematic error image - psImage *subMask = convolveSubsetAlloc(mask, border, threaded); // Mask + psImage *subVariance = variance ? psImageSubset(variance, border) : NULL; // Variance map + psImage *subSys = sys ? psImageSubset(sys, border) : NULL; // Systematic error image + psImage *subMask = mask ? psImageSubset(mask, border) : NULL; // Mask // XXX Can trim this a little by combining the convolution: only have to take the FFT of the kernel once @@ -310,7 +269,7 @@ psImage *convSys = subSys ? psImageConvolveFFT(NULL, subSys, subMask, maskVal, kernel) : NULL; // Conv sys - convolveSubsetFree(variance, subVariance, threaded); - convolveSubsetFree(sys, subSys, threaded); - convolveSubsetFree(mask, subMask, threaded); + psFree(subVariance); + psFree(subSys); + psFree(subMask); // Now, we have to stick it in where it belongs @@ -420,15 +379,13 @@ if (box > 0) { int colMin = region.x0, colMax = region.x1, rowMin = region.y0, rowMax = region.y1; // Bounds - bool threaded = pmSubtractionThreaded(); // Are we running threaded? - psRegion region = psRegionSet(colMin - box, colMax + box, rowMin - box, rowMax + box); // Region to convolve - psImage *image = convolveSubsetAlloc(subMask, region, threaded); // Mask to convolve + psImage *image = subMask ? psImageSubset(subMask, region) : NULL; // Mask to convolve psImage *convolved = psImageConvolveMask(NULL, image, subBad, subConvBad, -box, box, -box, box); // Convolved subtraction mask - convolveSubsetFree(subMask, image, threaded); + psFree(image); psAssert(convolved->numCols - 2 * box == colMax - colMin, "Bad number of columns"); @@ -653,7 +610,7 @@ float value = 0.0; // Value of convolved pixel int uMin = x - size, uMax = x + size; // Range for u - psF32 *xKernelData = xKernel->data.F32; // Kernel values + psF32 *xKernelData = &xKernel->data.F32[xKernel->n - 1]; // Kernel values psF32 *imageData = &image->kernel[y][uMin]; // Image values - for (int u = uMin; u <= uMax; u++, xKernelData++, imageData++) { + for (int u = uMin; u <= uMax; u++, xKernelData--, imageData++) { value += *xKernelData * *imageData; } @@ -668,7 +625,7 @@ float value = 0.0; // Value of convolved pixel int vMin = y - size, vMax = y + size; // Range for v - psF32 *yKernelData = yKernel->data.F32; // Kernel values + psF32 *yKernelData = &yKernel->data.F32[yKernel->n - 1]; // Kernel values psF32 *imageData = &temp->kernel[x][vMin]; // Image values; NOTE: wrong way! - for (int v = vMin; v <= vMax; v++, yKernelData++, imageData++) { + for (int v = vMin; v <= vMax; v++, yKernelData--, imageData++) { value += *yKernelData * *imageData; } Index: branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionEquation.c =================================================================== --- branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionEquation.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionEquation.c (revision 24557) @@ -15,5 +15,7 @@ #include "pmSubtractionVisual.h" -//#define TESTING +//#define TESTING // TESTING output for debugging; may not work with threads! + +#define USE_VARIANCE // Include variance in equation? ////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -31,5 +33,9 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { - sum += image1->kernel[y][x] * image2->kernel[y][x] / 1.0; // variance->kernel[y][x]; + double value = image1->kernel[y][x] * image2->kernel[y][x]; +#ifdef USE_VARIANCE + value /= variance->kernel[y][x]; +#endif + sum += value; } } @@ -195,5 +201,9 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { - sumC += conv->kernel[y][x] / 1.0; // variance->kernel[y][x]; + double value = conv->kernel[y][x]; +#ifdef USE_VARIANCE + value /= variance->kernel[y][x]; +#endif + sumC += value; } } @@ -218,5 +228,8 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { - double invNoise2 = 1.0 / 1.0; // variance->kernel[y][x]; + double invNoise2 = 1.0; +#ifdef USE_VARIANCE + invNoise2 /= variance->kernel[y][x]; +#endif double value = input->kernel[y][x] * invNoise2; sumI += value; @@ -277,5 +290,9 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { - sumTC += target->kernel[y][x] * conv->kernel[y][x] / 1.0; // variance->kernel[y][x]; + double value = target->kernel[y][x] * conv->kernel[y][x]; +#ifdef USE_VARIANCE + value /= variance->kernel[y][x]; +#endif + sumTC += value; } } @@ -297,5 +314,8 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { - float value = target->kernel[y][x] / 1.0; // variance->kernel[y][x]; + double value = target->kernel[y][x]; +#ifdef USE_VARIANCE + value /= variance->kernel[y][x]; +#endif sumIT += value * input->kernel[y][x]; sumT += value; @@ -366,5 +386,9 @@ for (int y = - footprint; y <= footprint; y++) { for (int x = - footprint; x <= footprint; x++) { - sumC += conv->kernel[y][x] / 1.0; // variance->kernel[y][x]; + double value = conv->kernel[y][x]; +#ifdef USE_VARIANCE + value /= variance->kernel[y][x]; +#endif + sumC += value; } } Index: branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionKernels.c =================================================================== --- branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionKernels.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionKernels.c (revision 24557) @@ -145,10 +145,8 @@ // Calculate moments - double sum = 0.0; // Sum of kernel component, for normalisation double moment = 0.0; // Moment, for penalty for (int v = -size, y = 0; v <= size; v++, y++) { for (int u = -size, x = 0; u <= size; u++, x++) { double value = xKernel->data.F32[x] * yKernel->data.F32[y]; // Value of kernel - sum += value; moment += value * (PS_SQR(u) + PS_SQR(v)); } @@ -163,8 +161,8 @@ } } - sum = 1.0 / sum; + sum = 1.0 / sqrt(sum); psBinaryOp(xKernel, xKernel, "*", psScalarAlloc(sum, PS_TYPE_F32)); psBinaryOp(yKernel, yKernel, "*", psScalarAlloc(sum, PS_TYPE_F32)); - moment *= sum; + moment *= PS_SQR(sum); } Index: branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionMask.c =================================================================== --- branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionMask.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionMask.c (revision 24557) @@ -6,4 +6,5 @@ #include +#include "pmErrorCodes.h" #include "pmSubtraction.h" #include "pmSubtractionKernels.h" @@ -78,5 +79,5 @@ } if (numBad > badFrac * numCols * numRows) { - psError(PS_ERR_BAD_PARAMETER_VALUE, true, + psError(PM_ERR_SMALL_AREA, true, "Fraction of bad pixels (%d/%d=%f) exceeds limit (%f)\n", numBad, numCols * numRows, (float)numBad/(float)(numCols * numRows), badFrac); Index: branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionMatch.c =================================================================== --- branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionMatch.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/imcombine/pmSubtractionMatch.c (revision 24557) @@ -246,5 +246,5 @@ badFrac, mode); // Subtraction mask if (!subMask) { - psError(PS_ERR_UNKNOWN, false, "Unable to generate subtraction mask."); + psError(psErrorCodeLast(), false, "Unable to generate subtraction mask."); psFree(kernels); psFree(regions); @@ -337,5 +337,5 @@ PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(optThreshold, 1.0, false); } - PS_ASSERT_INT_POSITIVE(iter, false); + PS_ASSERT_INT_NONNEGATIVE(iter, false); PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); @@ -379,5 +379,5 @@ badFrac, subMode); if (!subMask) { - psError(PS_ERR_UNKNOWN, false, "Unable to generate subtraction mask."); + psError(psErrorCodeLast(), false, "Unable to generate subtraction mask."); goto MATCH_ERROR; } Index: branches/eam_branches/20090522/psModules/src/objects/Makefile.am =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/Makefile.am (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/Makefile.am (revision 24557) @@ -38,4 +38,5 @@ pmSourceIO_PS1_CAL_0.c \ pmSourceIO_CMF_PS1_V1.c \ + pmSourceIO_CMF_PS1_V2.c \ pmSourceIO_MatchedRefs.c \ pmSourcePlots.c \ Index: branches/eam_branches/20090522/psModules/src/objects/pmFootprintArraysMerge.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmFootprintArraysMerge.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmFootprintArraysMerge.c (revision 24557) @@ -28,18 +28,26 @@ */ psArray *pmFootprintArraysMerge(const psArray *footprints1, // one set of footprints - const psArray *footprints2, // the other set - const int includePeaks) { // which peaks to set? 0x1 => footprints1, 0x2 => 2 - assert (footprints1->n == 0 || pmFootprintTest(footprints1->data[0])); - assert (footprints2->n == 0 || pmFootprintTest(footprints2->data[0])); + const psArray *footprints2, // the other set + const int includePeaks // which peaks to set? 0x1 => footprints1, 0x2 => 2 + ) +{ + if (!footprints1 && !footprints2) { + // No footprints in merged array + return psArrayAllocEmpty(0); + } - if (footprints1->n == 0 || footprints2->n == 0) { // nothing to do but put copies on merged - const psArray *old = (footprints1->n == 0) ? footprints2 : footprints1; + assert(!footprints1 || footprints1->n == 0 || pmFootprintTest(footprints1->data[0])); + assert(!footprints2 || footprints2->n == 0 || pmFootprintTest(footprints2->data[0])); - psArray *merged = psArrayAllocEmpty(old->n); - for (int i = 0; i < old->n; i++) { - psArrayAdd(merged, 1, old->data[i]); - } - - return merged; + if (!footprints1 || footprints1->n == 0 || !footprints2 || footprints2->n == 0) { + // nothing to do but put copies on merged + const psArray *old = (!footprints1 || footprints1->n == 0) ? footprints2 : footprints1; + + psArray *merged = psArrayAllocEmpty(old->n); + for (int i = 0; i < old->n; i++) { + psArrayAdd(merged, 1, old->data[i]); + } + + return merged; } /* @@ -48,14 +56,14 @@ */ { - pmFootprint *fp1 = footprints1->data[0]; - pmFootprint *fp2 = footprints2->data[0]; - if (fp1->region.x0 != fp2->region.x0 || - fp1->region.x1 != fp2->region.x1 || - fp1->region.y0 != fp2->region.y0 || - fp1->region.y1 != fp2->region.y1) { - psError(PS_ERR_BAD_PARAMETER_SIZE, true, - "The two pmFootprint arrays correspnond to different-sized regions"); - return NULL; - } + pmFootprint *fp1 = footprints1->data[0]; + pmFootprint *fp2 = footprints2->data[0]; + if (fp1->region.x0 != fp2->region.x0 || + fp1->region.x1 != fp2->region.x1 || + fp1->region.y0 != fp2->region.y0 || + fp1->region.y1 != fp2->region.y1) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, + "The two pmFootprint arrays correspnond to different-sized regions"); + return NULL; + } } /* @@ -72,17 +80,17 @@ * Now assign the peaks appropriately. We could do this more efficiently * using idImage (which we just freed), but this is easy and probably fast enough - */ + */ if (includePeaks & 0x1) { - psArray *peaks = pmFootprintArrayToPeaks(footprints1); - pmFootprintsAssignPeaks(merged, peaks); - psFree(peaks); + psArray *peaks = pmFootprintArrayToPeaks(footprints1); + pmFootprintsAssignPeaks(merged, peaks); + psFree(peaks); } if (includePeaks & 0x2) { - psArray *peaks = pmFootprintArrayToPeaks(footprints2); - pmFootprintsAssignPeaks(merged, peaks); - psFree(peaks); + psArray *peaks = pmFootprintArrayToPeaks(footprints2); + pmFootprintsAssignPeaks(merged, peaks); + psFree(peaks); } - + return merged; } Index: branches/eam_branches/20090522/psModules/src/objects/pmMoments.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmMoments.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmMoments.c (revision 24557) @@ -51,4 +51,5 @@ tmp->Sky = 0.0; tmp->nPixels = 0; + tmp->SN = 0; psTrace("psModules.objects", 10, "---- %s() end ----\n", __func__); Index: branches/eam_branches/20090522/psModules/src/objects/pmSource.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmSource.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmSource.c (revision 24557) @@ -480,14 +480,14 @@ if (!psVectorStats (stats, tmpSx, NULL, NULL, 0)) { - psError(PS_ERR_UNKNOWN, false, "failed to measure Sx stats"); + psError(PS_ERR_UNKNOWN, false, "failed to measure Sx stats"); return (emptyClump); - } + } psfClump.X = stats->clippedMean; psfClump.dX = stats->clippedStdev; if (!psVectorStats (stats, tmpSy, NULL, NULL, 0)) { - psError(PS_ERR_UNKNOWN, false, "failed to measure Sy stats"); + psError(PS_ERR_UNKNOWN, false, "failed to measure Sy stats"); return (emptyClump); - } + } psfClump.Y = stats->clippedMean; psfClump.dY = stats->clippedStdev; @@ -548,12 +548,12 @@ pmSource *source = (pmSource *) sources->data[i]; - // psf clumps are found for image subregions: - // skip sources not in this region + // psf clumps are found for image subregions: + // skip sources not in this region if (source->peak->x < region->x0) continue; if (source->peak->x >= region->x1) continue; if (source->peak->y < region->y0) continue; - if (source->peak->y >= region->y1) continue; - - // should be set by pmSourceAlloc + if (source->peak->y >= region->y1) continue; + + // should be set by pmSourceAlloc psAssert (source->type == PM_SOURCE_TYPE_UNKNOWN, "source type was not init-ed?"); @@ -561,5 +561,5 @@ if (!source->moments) { source->type = PM_SOURCE_TYPE_STAR; - psAssert (source->mode & noMoments, "why is this source missing moments?"); + psAssert (source->mode & noMoments, "why is this source missing moments?"); Nstar++; continue; @@ -596,36 +596,38 @@ } - // likely defect (too small to be stellar) (push out to 3 sigma) - // low S/N objects which are small are probably stellar - // only set candidate defects if - // XXX these limits are quite arbitrary - if ((sigX < 0.05) || (sigY < 0.05)) { - source->type = PM_SOURCE_TYPE_DEFECT; - source->mode |= PM_SOURCE_MODE_DEFECT; - Ncr ++; - continue; - } - - // likely unsaturated extended source (too large to be stellar) - if ((sigX > (clump.X + 3*clump.dX)) || (sigY > (clump.Y + 3*clump.dY))) { - source->type = PM_SOURCE_TYPE_EXTENDED; - Next ++; - continue; - } - - // the rest are probable stellar objects - starsn_moments->data.F32[starsn_moments->n] = source->moments->SN; - starsn_moments->n ++; - starsn_peaks->data.F32[starsn_peaks->n] = source->peak->SN; - starsn_peaks->n ++; - Nstar ++; - - // PSF star (within 1.5 sigma of clump center, S/N > limit) - psF32 radius = hypot ((sigX-clump.X)/clump.dX, (sigY-clump.Y)/clump.dY); - if ((source->moments->SN > PSF_SN_LIM) && (radius < PSF_CLUMP_NSIGMA)) { - source->type = PM_SOURCE_TYPE_STAR; - source->mode |= PM_SOURCE_MODE_PSFSTAR; - Npsf ++; - continue; + // The following determinations require the use of moments + if (!(source->mode & noMoments)) { + // likely defect (too small to be stellar) (push out to 3 sigma) + // low S/N objects which are small are probably stellar + // XXX these limits are quite arbitrary + if (sigX < 0.05 || sigY < 0.05) { + source->type = PM_SOURCE_TYPE_DEFECT; + source->mode |= PM_SOURCE_MODE_DEFECT; + Ncr ++; + continue; + } + + // likely unsaturated extended source (too large to be stellar) + if (sigX > clump.X + 3*clump.dX || sigY > clump.Y + 3*clump.dY) { + source->type = PM_SOURCE_TYPE_EXTENDED; + Next ++; + continue; + } + + // the rest are probable stellar objects + starsn_moments->data.F32[starsn_moments->n] = source->moments->SN; + starsn_moments->n ++; + starsn_peaks->data.F32[starsn_peaks->n] = source->peak->SN; + starsn_peaks->n ++; + Nstar ++; + + // PSF star (within 1.5 sigma of clump center, S/N > limit) + psF32 radius = hypot ((sigX-clump.X)/clump.dX, (sigY-clump.Y)/clump.dY); + if ((source->moments->SN > PSF_SN_LIM) && (radius < PSF_CLUMP_NSIGMA)) { + source->type = PM_SOURCE_TYPE_STAR; + source->mode |= PM_SOURCE_MODE_PSFSTAR; + Npsf ++; + continue; + } } @@ -642,8 +644,8 @@ if (!psVectorStats (stats, starsn_moments, NULL, NULL, 0)) { - psError(PS_ERR_UNKNOWN, false, "failed to measure SN / moments stats"); - psFree (stats); - psFree (starsn_peaks); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to measure SN / moments stats"); + psFree (stats); + psFree (starsn_peaks); + return false; } psLogMsg ("pmObjects", 3, "SN range (moments): %f - %f\n", stats->min, stats->max); @@ -656,8 +658,8 @@ stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX); if (!psVectorStats (stats, starsn_peaks, NULL, NULL, 0)) { - psError(PS_ERR_UNKNOWN, false, "failed to measure SN / moments stats"); - psFree (stats); - psFree (starsn_peaks); - return false; + psError(PS_ERR_UNKNOWN, false, "failed to measure SN / moments stats"); + psFree (stats); + psFree (starsn_peaks); + return false; } psLogMsg ("psModules.objects", 3, "SN range (peaks) : %f - %f (%ld)\n", Index: branches/eam_branches/20090522/psModules/src/objects/pmSourceIO.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmSourceIO.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmSourceIO.c (revision 24557) @@ -496,4 +496,7 @@ status = pmSourcesWrite_CMF_PS1_V1 (file->fits, readout, sources, file->header, outhead, dataname); } + if (!strcmp (exttype, "PS1_V2")) { + status = pmSourcesWrite_CMF_PS1_V2 (file->fits, readout, sources, file->header, outhead, dataname); + } if (xsrcname) { if (!strcmp (exttype, "PS1_DEV_1")) { @@ -506,4 +509,7 @@ status = pmSourcesWrite_CMF_PS1_V1_XSRC (file->fits, sources, xsrcname, recipe); } + if (!strcmp (exttype, "PS1_V2")) { + status = pmSourcesWrite_CMF_PS1_V2_XSRC (file->fits, sources, xsrcname, recipe); + } } if (xfitname) { @@ -517,4 +523,7 @@ status = pmSourcesWrite_CMF_PS1_V1_XFIT (file->fits, sources, xfitname); } + if (!strcmp (exttype, "PS1_V2")) { + status = pmSourcesWrite_CMF_PS1_V2_XFIT (file->fits, sources, xfitname); + } } if (!status) { @@ -944,4 +953,7 @@ sources = pmSourcesRead_CMF_PS1_V1 (file->fits, hdu->header); } + if (!strcmp (exttype, "PS1_V2")) { + sources = pmSourcesRead_CMF_PS1_V2 (file->fits, hdu->header); + } } Index: branches/eam_branches/20090522/psModules/src/objects/pmSourceIO.h =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmSourceIO.h (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmSourceIO.h (revision 24557) @@ -39,4 +39,8 @@ bool pmSourcesWrite_CMF_PS1_V1_XFIT (psFits *fits, psArray *sources, char *extname); +bool pmSourcesWrite_CMF_PS1_V2 (psFits *fits, pmReadout *readout, psArray *sources, psMetadata *imageHeader, psMetadata *tableHeader, char *extname); +bool pmSourcesWrite_CMF_PS1_V2_XSRC (psFits *fits, psArray *sources, char *extname, psMetadata *recipe); +bool pmSourcesWrite_CMF_PS1_V2_XFIT (psFits *fits, psArray *sources, char *extname); + bool pmSource_CMF_WritePHU (const pmFPAview *view, pmFPAfile *file, pmConfig *config); @@ -48,4 +52,5 @@ psArray *pmSourcesRead_PS1_CAL_0 (psFits *fits, psMetadata *header); psArray *pmSourcesRead_CMF_PS1_V1 (psFits *fits, psMetadata *header); +psArray *pmSourcesRead_CMF_PS1_V2 (psFits *fits, psMetadata *header); bool pmSourcesWritePSFs (psArray *sources, char *filename); Index: branches/eam_branches/20090522/psModules/src/objects/pmSourceIO_CMF_PS1_V2.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmSourceIO_CMF_PS1_V2.c (revision 24557) +++ branches/eam_branches/20090522/psModules/src/objects/pmSourceIO_CMF_PS1_V2.c (revision 24557) @@ -0,0 +1,625 @@ +/** @file pmSourceIO.c + * + * @author EAM, IfA + * + * @version $Revision: 1.3 $ $Name: not supported by cvs2svn $ + * @date $Date: 2009-02-18 02:44:19 $ + * + * Copyright 2004 Maui High Performance Computing Center, University of Hawaii + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmConfig.h" +#include "pmDetrendDB.h" + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmFPALevel.h" +#include "pmFPAview.h" +#include "pmFPAfile.h" + +#include "pmSpan.h" +#include "pmFootprint.h" +#include "pmPeaks.h" +#include "pmMoments.h" +#include "pmGrowthCurve.h" +#include "pmResiduals.h" +#include "pmTrend2D.h" +#include "pmPSF.h" +#include "pmModel.h" +#include "pmSource.h" +#include "pmModelClass.h" +#include "pmSourceIO.h" + +// panstarrs-style FITS table output (header + table in 1st extension) +// this format consists of a header derived from the image header +// followed by a zero-size matrix, followed by the table data + +bool pmSourcesWrite_CMF_PS1_V2 (psFits *fits, pmReadout *readout, psArray *sources, + psMetadata *imageHeader, psMetadata *tableHeader, char *extname) +{ + PS_ASSERT_PTR_NON_NULL(fits, false); + PS_ASSERT_PTR_NON_NULL(sources, false); + PS_ASSERT_PTR_NON_NULL(extname, false); + + psArray *table; + psMetadata *row; + int i; + psF32 *PAR, *dPAR; + psEllipseAxes axes; + psF32 xPos, yPos; + psF32 xErr, yErr; + psF32 errMag, chisq, apRadius; + psS32 nPix, nDOF; + + pmChip *chip = readout->parent->parent; + pmFPA *fpa = chip->parent; + + bool status1 = false; + bool status2 = false; + float magOffset = NAN; + float exptime = psMetadataLookupF32 (&status1, fpa->concepts, "FPA.EXPOSURE"); + float zeropt = psMetadataLookupF32 (&status2, imageHeader, "ZPT_OBS"); + float zeroptErr = psMetadataLookupF32 (&status2, imageHeader, "ZPT_ERR"); + if (status1 && status2 && (exptime > 0.0)) { + magOffset = zeropt + 2.5*log10(exptime); + } + + // if the sequence is defined, write these in seq order; otherwise + // write them in S/N order: + if (sources->n > 0) { + pmSource *source = (pmSource *) sources->data[0]; + if (source->seq == -1) { + // let's write these out in S/N order + sources = psArraySort (sources, pmSourceSortBySN); + } else { + sources = psArraySort (sources, pmSourceSortBySeq); + } + } + + table = psArrayAllocEmpty (sources->n); + + // we write out PSF-fits for all sources, regardless of quality. the source flags tell us the state + // by the time we call this function, all values should be assigned. let's use asserts to be sure in some cases. + for (i = 0; i < sources->n; i++) { + pmSource *source = (pmSource *) sources->data[i]; + + // If source->seq is -1, source was generated in this analysis. If source->seq is + // not -1, source was read from elsewhere: in the latter case, preserve the source + // ID. source.seq is used instead of source.id since the latter is a const + // generated on Alloc, and would thus be wrong for read in sources. + if (source->seq == -1) { + source->seq = i; + } + + // no difference between PSF and non-PSF model + pmModel *model = source->modelPSF; + + if (model != NULL) { + PAR = model->params->data.F32; + dPAR = model->dparams->data.F32; + xPos = PAR[PM_PAR_XPOS]; + yPos = PAR[PM_PAR_YPOS]; + if (source->mode & PM_SOURCE_MODE_NONLINEAR_FIT) { + xErr = dPAR[PM_PAR_XPOS]; + yErr = dPAR[PM_PAR_YPOS]; + } else { + // in linear-fit mode, there is no error on the centroid + xErr = source->peak->dx; + yErr = source->peak->dy; + } + if (isfinite(PAR[PM_PAR_SXX]) && isfinite(PAR[PM_PAR_SXX]) && isfinite(PAR[PM_PAR_SXX])) { + axes = pmPSF_ModelToAxes (PAR, 20.0); + } else { + axes.major = NAN; + axes.minor = NAN; + axes.theta = NAN; + } + chisq = model->chisq; + nDOF = model->nDOF; + nPix = model->nPix; + apRadius = model->radiusFit; // XXX should we really use the fitRadius for aperture Radius? + errMag = model->dparams->data.F32[PM_PAR_I0] / model->params->data.F32[PM_PAR_I0]; + } else { + xPos = source->peak->xf; + yPos = source->peak->yf; + xErr = source->peak->dx; + yErr = source->peak->dy; + axes.major = NAN; + axes.minor = NAN; + axes.theta = NAN; + chisq = NAN; + nDOF = 0; + nPix = 0; + apRadius = NAN; + errMag = NAN; + } + + float calMag = isfinite(magOffset) ? source->psfMag + magOffset : NAN; + float peakMag = (source->peak->flux > 0) ? -2.5*log10(source->peak->flux) : NAN; + psS16 nImageOverlap = 1; + + psSphere ptSky = {0.0, 0.0, 0.0, 0.0}; + float posAngle = 0.0; + float pltScale = 0.0; + pmSourceLocalAstrometry (&ptSky, &posAngle, &pltScale, chip, xPos, yPos); + + row = psMetadataAlloc (); + psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET", PS_DATA_U32, "IPP detection identifier index", source->seq); + psMetadataAdd (row, PS_LIST_TAIL, "X_PSF", PS_DATA_F32, "PSF x coordinate", xPos); + psMetadataAdd (row, PS_LIST_TAIL, "Y_PSF", PS_DATA_F32, "PSF y coordinate", yPos); + psMetadataAdd (row, PS_LIST_TAIL, "X_PSF_SIG", PS_DATA_F32, "Sigma in PSF x coordinate", xErr); // XXX this is only measured for non-linear fits + psMetadataAdd (row, PS_LIST_TAIL, "Y_PSF_SIG", PS_DATA_F32, "Sigma in PSF y coordinate", yErr); // XXX this is only measured for non-linear fits + psMetadataAdd (row, PS_LIST_TAIL, "POSANGLE", PS_DATA_F32, "position angle at source (degrees)", posAngle*PS_DEG_RAD); + psMetadataAdd (row, PS_LIST_TAIL, "PLTSCALE", PS_DATA_F32, "plate scale at source (arcsec/pixel)", pltScale*PS_DEG_RAD*3600.0); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_MAG", PS_DATA_F32, "PSF fit instrumental magnitude", source->psfMag); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_INST_MAG_SIG", PS_DATA_F32, "Sigma of PSF instrumental magnitude", errMag); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG", PS_DATA_F32, "magnitude in standard aperture", source->apMag); + psMetadataAdd (row, PS_LIST_TAIL, "AP_MAG_RADIUS", PS_DATA_F32, "radius used for aperture mags", apRadius); + psMetadataAdd (row, PS_LIST_TAIL, "PEAK_FLUX_AS_MAG", PS_DATA_F32, "Peak flux expressed as magnitude", peakMag); + psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG", PS_DATA_F32, "PSF Magnitude using supplied calibration", calMag); + psMetadataAdd (row, PS_LIST_TAIL, "CAL_PSF_MAG_SIG", PS_DATA_F32, "measured scatter of zero point calibration", zeroptErr); + psMetadataAdd (row, PS_LIST_TAIL, "RA_PSF", PS_DATA_F64, "PSF RA coordinate (degrees)", ptSky.r*PS_DEG_RAD); + psMetadataAdd (row, PS_LIST_TAIL, "DEC_PSF", PS_DATA_F64, "PSF DEC coordinate (degrees)", ptSky.d*PS_DEG_RAD); + psMetadataAdd (row, PS_LIST_TAIL, "SKY", PS_DATA_F32, "Sky level", source->sky); + psMetadataAdd (row, PS_LIST_TAIL, "SKY_SIGMA", PS_DATA_F32, "Sigma of sky level", source->skyErr); + + psMetadataAdd (row, PS_LIST_TAIL, "PSF_CHISQ", PS_DATA_F32, "Chisq of PSF-fit", chisq); + psMetadataAdd (row, PS_LIST_TAIL, "CR_NSIGMA", PS_DATA_F32, "Nsigma deviations from PSF to CF", source->crNsigma); + psMetadataAdd (row, PS_LIST_TAIL, "EXT_NSIGMA", PS_DATA_F32, "Nsigma deviations from PSF to EXT", source->extNsigma); + + psMetadataAdd (row, PS_LIST_TAIL, "PSF_MAJOR", PS_DATA_F32, "PSF width (major axis)", axes.major); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_MINOR", PS_DATA_F32, "PSF width (minor axis)", axes.minor); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_THETA", PS_DATA_F32, "PSF orientation angle", axes.theta); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_QF", PS_DATA_F32, "PSF coverage/quality factor", source->pixWeight); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_NDOF", PS_DATA_S32, "degrees of freedom", nDOF); + psMetadataAdd (row, PS_LIST_TAIL, "PSF_NPIX", PS_DATA_S32, "number of pixels in fit", nPix); + + // distinguish moments measure from window vs S/N > XX ?? + float mxx = source->moments ? source->moments->Mxx : NAN; + float mxy = source->moments ? source->moments->Mxy : NAN; + float myy = source->moments ? source->moments->Myy : NAN; + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XX", PS_DATA_F32, "second moments (X^2)", mxx); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_XY", PS_DATA_F32, "second moments (X*Y)", mxy); + psMetadataAdd (row, PS_LIST_TAIL, "MOMENTS_YY", PS_DATA_F32, "second moments (Y*Y)", myy); + + psMetadataAdd (row, PS_LIST_TAIL, "FLAGS", PS_DATA_U32, "psphot analysis flags", source->mode); + + // XXX not sure how to get this : need to load Nimages with weight? + psMetadataAdd (row, PS_LIST_TAIL, "N_FRAMES", PS_DATA_U16, "Number of frames overlapping source center", nImageOverlap); + psMetadataAdd (row, PS_LIST_TAIL, "PADDING", PS_DATA_S16, "padding", 0); + + psArrayAdd (table, 100, row); + psFree (row); + + // EXT_NSIGMA will be NAN if: 1) contour ellipse is imaginary; 2) source is not + // subtracted + + // CR_NSIGMA will be NAN if: 1) source is not subtracted; 2) source is on the image + // edge; 3) any pixels in the 3x3 peak region are masked; + } + + psMetadata *header = psMetadataCopy(NULL, tableHeader); + pmSourceMasksHeader(header); + + if (table->n == 0) { + psFitsWriteBlank(fits, header, extname); + psFree(table); + psFree(header); + return true; + } + + psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname); + if (!psFitsWriteTable(fits, header, table, extname)) { + psError(PS_ERR_IO, false, "writing ext data %s\n", extname); + psFree(table); + psFree(header); + return false; + } + psFree(table); + psFree(header); + + return true; +} + +// read in a readout from the fits file +psArray *pmSourcesRead_CMF_PS1_V2 (psFits *fits, psMetadata *header) +{ + PS_ASSERT_PTR_NON_NULL(fits, false); + PS_ASSERT_PTR_NON_NULL(header, false); + + bool status; + psF32 *PAR, *dPAR; + psEllipseAxes axes; + + // define PSF model type + // XXX need to carry the extra model parameters + int modelType = pmModelClassGetType ("PS_MODEL_GAUSS"); + + char *PSF_NAME = psMetadataLookupStr (&status, header, "PSF_NAME"); + if (PSF_NAME != NULL) { + modelType = pmModelClassGetType (PSF_NAME); + } + assert (modelType > -1); + + // We get the size of the table, and allocate the array of sources first because the table + // is large and ephemeral --- when the table gets blown away, whatever is allocated after + // the table is read blocks the free. In fact, it's better to read the table row by row. + long numSources = psFitsTableSize(fits); // Number of sources in table + psArray *sources = psArrayAlloc(numSources); // Array of sources, to return + + // convert the table to the pmSource entriesa + for (int i = 0; i < numSources; i++) { + psMetadata *row = psFitsReadTableRow(fits, i); // Table row + + pmSource *source = pmSourceAlloc (); + pmModel *model = pmModelAlloc (modelType); + source->modelPSF = model; + source->type = PM_SOURCE_TYPE_STAR; // XXX this should be added to the flags + + // NOTE: A SEGV here because "model" is NULL is probably caused by not initialising the models. + PAR = model->params->data.F32; + dPAR = model->dparams->data.F32; + + source->seq = psMetadataLookupU32 (&status, row, "IPP_IDET"); + PAR[PM_PAR_XPOS] = psMetadataLookupF32 (&status, row, "X_PSF"); + PAR[PM_PAR_YPOS] = psMetadataLookupF32 (&status, row, "Y_PSF"); + dPAR[PM_PAR_XPOS] = psMetadataLookupF32 (&status, row, "X_PSF_SIG"); + dPAR[PM_PAR_YPOS] = psMetadataLookupF32 (&status, row, "Y_PSF_SIG"); + axes.major = psMetadataLookupF32 (&status, row, "PSF_MAJOR"); + axes.minor = psMetadataLookupF32 (&status, row, "PSF_MINOR"); + axes.theta = psMetadataLookupF32 (&status, row, "PSF_THETA"); + + PAR[PM_PAR_SKY] = psMetadataLookupF32 (&status, row, "SKY"); + dPAR[PM_PAR_SKY] = psMetadataLookupF32 (&status, row, "SKY_SIGMA"); + source->sky = PAR[PM_PAR_SKY]; + source->skyErr = dPAR[PM_PAR_SKY]; + + // XXX use these to determine PAR[PM_PAR_I0]? + source->psfMag = psMetadataLookupF32 (&status, row, "PSF_INST_MAG"); + source->errMag = psMetadataLookupF32 (&status, row, "PSF_INST_MAG_SIG"); + source->apMag = psMetadataLookupF32 (&status, row, "AP_MAG"); + + // XXX this scaling is incorrect: does not include the 2 \pi AREA factor + PAR[PM_PAR_I0] = (isfinite(source->psfMag)) ? pow(10.0, -0.4*source->psfMag) : NAN; + dPAR[PM_PAR_I0] = (isfinite(source->psfMag)) ? PAR[PM_PAR_I0] * source->errMag : NAN; + + pmPSF_AxesToModel (PAR, axes); + + float peakMag = psMetadataLookupF32 (&status, row, "PEAK_FLUX_AS_MAG"); + float peakFlux = (isfinite(peakMag)) ? pow(10.0, -0.4*peakMag) : NAN; + + // recreate the peak to match (xPos, yPos) +/- (xErr, yErr) + source->peak = pmPeakAlloc(PAR[PM_PAR_XPOS], PAR[PM_PAR_YPOS], peakFlux, PM_PEAK_LONE); + source->peak->flux = peakFlux; + source->peak->dx = dPAR[PM_PAR_XPOS]; + source->peak->dy = dPAR[PM_PAR_YPOS]; + + source->pixWeight = psMetadataLookupF32 (&status, row, "PSF_QF"); + source->crNsigma = psMetadataLookupF32 (&status, row, "CR_NSIGMA"); + source->extNsigma = psMetadataLookupF32 (&status, row, "EXT_NSIGMA"); + + // note that some older versions used PSF_PROBABILITY: this was not well defined. + model->chisq = psMetadataLookupF32 (&status, row, "PSF_CHISQ"); + model->nDOF = psMetadataLookupS32 (&status, row, "PSF_NDOF"); + model->nPix = psMetadataLookupS32 (&status, row, "PSF_NPIX"); + model->radiusFit = psMetadataLookupS32 (&status, row, "AP_MAG_RADIUS"); + + source->moments = pmMomentsAlloc (); + source->moments->Mxx = psMetadataLookupF32 (&status, row, "MOMENTS_XX"); + source->moments->Mxy = psMetadataLookupF32 (&status, row, "MOMENTS_XY"); + source->moments->Myy = psMetadataLookupF32 (&status, row, "MOMENTS_YY"); + + source->mode = psMetadataLookupU32 (&status, row, "FLAGS"); + assert (status); + + sources->data[i] = source; + psFree(row); + } + + return sources; +} + +// XXX this layout is still the same as PS1_DEV_1 +bool pmSourcesWrite_CMF_PS1_V2_XSRC (psFits *fits, psArray *sources, char *extname, psMetadata *recipe) +{ + + bool status; + psArray *table; + psMetadata *row; + psF32 *PAR, *dPAR; + psF32 xPos, yPos; + psF32 xErr, yErr; + + // create a header to hold the output data + psMetadata *outhead = psMetadataAlloc (); + + // write the links to the image header + psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "xsrc table extension", extname); + + // let's write these out in S/N order + sources = psArraySort (sources, pmSourceSortBySN); + + table = psArrayAllocEmpty (sources->n); + + // which extended source analyses should we perform? + bool doPetrosian = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_PETROSIAN"); + bool doIsophotal = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ISOPHOTAL"); + bool doAnnuli = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_ANNULI"); + bool doKron = psMetadataLookupBool (&status, recipe, "EXTENDED_SOURCE_KRON"); + + psVector *radialBinsLower = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.LOWER"); + psVector *radialBinsUpper = psMetadataLookupPtr (&status, recipe, "RADIAL.ANNULAR.BINS.UPPER"); + assert (radialBinsLower->n == radialBinsUpper->n); + + // we write out all sources, regardless of quality. the source flags tell us the state + for (int i = 0; i < sources->n; i++) { + // skip source if it is not a ext sourc + // XXX we have two places that extended source parameters are measured: + // psphotExtendedSources, which measures the aperture-like parameters and (potentially) the psf-convolved extended source models, + // psphotFitEXT, which does the simple extended source model fit (not psf-convolved) + // should we require both? + + pmSource *source = sources->data[i]; + + // skip sources without measurements + if (source->extpars == NULL) continue; + + // we require a PSF model fit (ignore the real crud) + pmModel *model = source->modelPSF; + if (model == NULL) continue; + + // XXX I need to split the extended models from the extended aperture measurements + PAR = model->params->data.F32; + dPAR = model->dparams->data.F32; + xPos = PAR[PM_PAR_XPOS]; + yPos = PAR[PM_PAR_YPOS]; + xErr = dPAR[PM_PAR_XPOS]; + yErr = dPAR[PM_PAR_YPOS]; + + row = psMetadataAlloc (); + + // XXX we are not writing out the mode (flags) or the type (psf, ext, etc) + psMetadataAdd (row, PS_LIST_TAIL, "IPP_IDET", PS_DATA_U32, "IPP detection identifier index", source->seq); + psMetadataAdd (row, PS_LIST_TAIL, "X_EXT", PS_DATA_F32, "EXT model x coordinate", xPos); + psMetadataAdd (row, PS_LIST_TAIL, "Y_EXT", PS_DATA_F32, "EXT model y coordinate", yPos); + psMetadataAdd (row, PS_LIST_TAIL, "X_EXT_SIG", PS_DATA_F32, "Sigma in EXT x coordinate", xErr); + psMetadataAdd (row, PS_LIST_TAIL, "Y_EXT_SIG", PS_DATA_F32, "Sigma in EXT y coordinate", yErr); + + // Petrosian measurements + // XXX insert header data: petrosian ref radius, flux ratio + if (doPetrosian) { + pmSourcePetrosianValues *petrosian = source->extpars->petrosian; + if (petrosian) { + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG", PS_DATA_F32, "Petrosian Magnitude", petrosian->mag); + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG_ERR", PS_DATA_F32, "Petrosian Magnitude Error", petrosian->magErr); + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS", PS_DATA_F32, "Petrosian Radius", petrosian->rad); + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS_ERR", PS_DATA_F32, "Petrosian Radius Error", petrosian->radErr); + } else { + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG", PS_DATA_F32, "Petrosian Magnitude", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_MAG_ERR", PS_DATA_F32, "Petrosian Magnitude Error", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS", PS_DATA_F32, "Petrosian Radius", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "PETRO_RADIUS_ERR", PS_DATA_F32, "Petrosian Radius Error", NAN); + } + } + + // Kron measurements + if (doKron) { + pmSourceKronValues *kron = source->extpars->kron; + if (kron) { + psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG", PS_DATA_F32, "Kron Magnitude", kron->mag); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG_ERR", PS_DATA_F32, "Kron Magnitude Error", kron->magErr); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS", PS_DATA_F32, "Kron Radius", kron->rad); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS_ERR", PS_DATA_F32, "Kron Radius Error", kron->radErr); + } else { + psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG", PS_DATA_F32, "Kron Magnitude", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_MAG_ERR", PS_DATA_F32, "Kron Magnitude Error", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS", PS_DATA_F32, "Kron Radius", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "KRON_RADIUS_ERR", PS_DATA_F32, "Kron Radius Error", NAN); + } + } + + // Isophot measurements + // XXX insert header data: isophotal level + if (doIsophotal) { + pmSourceIsophotalValues *isophot = source->extpars->isophot; + if (isophot) { + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG", PS_DATA_F32, "Isophot Magnitude", isophot->mag); + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG_ERR", PS_DATA_F32, "Isophot Magnitude Error", isophot->magErr); + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS", PS_DATA_F32, "Isophot Radius", isophot->rad); + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS_ERR", PS_DATA_F32, "Isophot Radius Error", isophot->radErr); + } else { + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG", PS_DATA_F32, "Isophot Magnitude", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_MAG_ERR", PS_DATA_F32, "Isophot Magnitude Error", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS", PS_DATA_F32, "Isophot Radius", NAN); + psMetadataAdd (row, PS_LIST_TAIL, "ISOPHOT_RADIUS_ERR", PS_DATA_F32, "Isophot Radius Error", NAN); + } + } + + // Flux Annuli + if (doAnnuli) { + pmSourceAnnuli *annuli = source->extpars->annuli; + if (annuli) { + psVector *fluxVal = annuli->flux; + psVector *fluxErr = annuli->fluxErr; + psVector *fluxVar = annuli->fluxVar; + + for (int j = 0; j < fluxVal->n; j++) { + char name[32]; + sprintf (name, "FLUX_VAL_R_%02d", j); + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux value in annulus", fluxVal->data.F32[j]); + sprintf (name, "FLUX_ERR_R_%02d", j); + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux error in annulus", fluxErr->data.F32[j]); + sprintf (name, "FLUX_VAR_R_%02d", j); + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux stdev in annulus", fluxVar->data.F32[j]); + } + } else { + for (int j = 0; j < radialBinsLower->n; j++) { + char name[32]; + sprintf (name, "FLUX_VAL_R_%02d", j); + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux value in annulus", NAN); + sprintf (name, "FLUX_ERR_R_%02d", j); + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux error in annulus", NAN); + sprintf (name, "FLUX_VAR_R_%02d", j); + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "flux stdev in annulus", NAN); + } + } + } + + psArrayAdd (table, 100, row); + psFree (row); + } + + if (table->n == 0) { + psFitsWriteBlank (fits, outhead, extname); + psFree (outhead); + psFree (table); + return true; + } + + psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname); + if (!psFitsWriteTable (fits, outhead, table, extname)) { + psError(PS_ERR_IO, false, "writing ext data %s\n", extname); + psFree (outhead); + psFree(table); + return false; + } + psFree (outhead); + psFree (table); + + return true; +} + +// XXX this layout is still the same as PS1_DEV_1 +bool pmSourcesWrite_CMF_PS1_V2_XFIT (psFits *fits, psArray *sources, char *extname) +{ + + psArray *table; + psMetadata *row; + psF32 *PAR, *dPAR; + psEllipseAxes axes; + psF32 xPos, yPos; + psF32 xErr, yErr; + char name[64]; + + // create a header to hold the output data + psMetadata *outhead = psMetadataAlloc (); + + // write the links to the image header + psMetadataAddStr (outhead, PS_LIST_TAIL, "EXTNAME", PS_META_REPLACE, "xsrc table extension", extname); + + // let's write these out in S/N order + sources = psArraySort (sources, pmSourceSortBySN); + + // we are writing one row per model; we need to write out same number of columns for each row: find the max Nparams + int nParamMax = 0; + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; + if (source->modelFits == NULL) continue; + for (int j = 0; j < source->modelFits->n; j++) { + pmModel *model = source->modelFits->data[j]; + assert (model); + nParamMax = PS_MAX (nParamMax, model->params->n); + } + } + + table = psArrayAllocEmpty (sources->n); + + // we write out all sources, regardless of quality. the source flags tell us the state + for (int i = 0; i < sources->n; i++) { + + pmSource *source = sources->data[i]; + + // XXX if no model fits are saved, write out modelEXT? + if (source->modelFits == NULL) continue; + + // We have multiple sources : need to flag the one used to subtract the light (the 'best' model) + for (int j = 0; j < source->modelFits->n; j++) { + + // choose the convolved EXT model, if available, otherwise the simple one + pmModel *model = source->modelFits->data[j]; + assert (model); + + PAR = model->params->data.F32; + dPAR = model->dparams->data.F32; + xPos = PAR[PM_PAR_XPOS]; + yPos = PAR[PM_PAR_YPOS]; + xErr = dPAR[PM_PAR_XPOS]; + yErr = dPAR[PM_PAR_YPOS]; + + axes = pmPSF_ModelToAxes (PAR, 20.0); + + row = psMetadataAlloc (); + + // XXX we are not writing out the mode (flags) or the type (psf, ext, etc) + psMetadataAddU32 (row, PS_LIST_TAIL, "IPP_IDET", 0, "IPP detection identifier index", source->seq); + psMetadataAddF32 (row, PS_LIST_TAIL, "X_EXT", 0, "EXT model x coordinate", xPos); + psMetadataAddF32 (row, PS_LIST_TAIL, "Y_EXT", 0, "EXT model y coordinate", yPos); + psMetadataAddF32 (row, PS_LIST_TAIL, "X_EXT_SIG", 0, "Sigma in EXT x coordinate", xErr); + psMetadataAddF32 (row, PS_LIST_TAIL, "Y_EXT_SIG", 0, "Sigma in EXT y coordinate", yErr); + psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_INST_MAG", 0, "EXT fit instrumental magnitude", model->mag); + psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_INST_MAG_SIG", 0, "Sigma of PSF instrumental magnitude", model->magErr); + + psMetadataAddF32 (row, PS_LIST_TAIL, "NPARAMS", 0, "number of model parameters", model->params->n); + psMetadataAddStr (row, PS_LIST_TAIL, "MODEL_TYPE", 0, "name of model", pmModelClassGetName (model->type)); + + // XXX these should be major and minor, not 'x' and 'y' + psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_WIDTH_MAJ", 0, "EXT width in x coordinate", axes.major); + psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_WIDTH_MIN", 0, "EXT width in y coordinate", axes.minor); + psMetadataAddF32 (row, PS_LIST_TAIL, "EXT_THETA", 0, "EXT orientation angle", axes.theta); + + // write out the other generic parameters + for (int k = 0; k < nParamMax; k++) { + if (k == PM_PAR_I0) continue; + if (k == PM_PAR_SKY) continue; + if (k == PM_PAR_XPOS) continue; + if (k == PM_PAR_YPOS) continue; + if (k == PM_PAR_SXX) continue; + if (k == PM_PAR_SXY) continue; + if (k == PM_PAR_SYY) continue; + + snprintf (name, 64, "EXT_PAR_%02d", k); + + if (k < model->params->n) { + psMetadataAdd (row, PS_LIST_TAIL, name, PS_DATA_F32, "", model->params->data.F32[k]); + } else { + psMetadataAddF32 (row, PS_LIST_TAIL, name, PS_DATA_F32, "", NAN); + } + } + + // XXX other parameters which may be set. + // XXX flag / value to define the model + // XXX write out the model type, fit status flags + + psArrayAdd (table, 100, row); + psFree (row); + } + } + + if (table->n == 0) { + psFitsWriteBlank (fits, outhead, extname); + psFree (outhead); + psFree (table); + return true; + } + + psTrace ("pmFPAfile", 5, "writing ext data %s\n", extname); + if (!psFitsWriteTable (fits, outhead, table, extname)) { + psError(PS_ERR_IO, false, "writing ext data %s\n", extname); + psFree (outhead); + psFree(table); + return false; + } + psFree (outhead); + psFree (table); + return true; +} Index: branches/eam_branches/20090522/psModules/src/objects/pmSourceIO_MatchedRefs.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmSourceIO_MatchedRefs.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmSourceIO_MatchedRefs.c (revision 24557) @@ -49,5 +49,4 @@ pmCell *cell = NULL; pmReadout *readout = NULL; - pmFPAview *view = pmFPAviewAlloc (0); // first, check if there are any matches to be written @@ -57,6 +56,6 @@ psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, "PSASTRO"); if (!status) { - psError(PS_ERR_UNKNOWN, true, "missing recipe PSASTRO in config data"); - return false; + psError(PS_ERR_UNKNOWN, true, "missing recipe PSASTRO in config data"); + return false; } @@ -65,4 +64,5 @@ psArray *table = psArrayAllocEmpty (0x1000); + pmFPAview *view = pmFPAviewAlloc (0); // this loop selects the matched stars for all chips @@ -70,58 +70,58 @@ psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process); if (!chip->process || !chip->file_exists) continue; - - char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); - while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) { + char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); + + while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) { psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process); if (!cell->process || !cell->file_exists) continue; - // process each of the readouts - // XXX there can only be one readout per chip, right? - while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) { - if (! readout->data_exists) continue; + // process each of the readouts + // XXX there can only be one readout per chip, right? + while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) { + if (! readout->data_exists) continue; - // select the raw objects for this readout - psArray *rawstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.RAWSTARS"); - if (rawstars == NULL) continue; + // select the raw objects for this readout + psArray *rawstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.RAWSTARS"); + if (rawstars == NULL) continue; - // select the raw objects for this readout - psArray *refstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.REFSTARS"); - if (refstars == NULL) continue; - psTrace ("psastro", 4, "Trying %ld refstars\n", refstars->n); + // select the raw objects for this readout + psArray *refstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.REFSTARS"); + if (refstars == NULL) continue; + psTrace ("psastro", 4, "Trying %ld refstars\n", refstars->n); - psArray *matches = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.MATCH"); - if (matches == NULL) continue; + psArray *matches = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.MATCH"); + if (matches == NULL) continue; - for (int i = 0; i < matches->n; i++) { - pmAstromMatch *match = matches->data[i]; + for (int i = 0; i < matches->n; i++) { + pmAstromMatch *match = matches->data[i]; - pmAstromObj *raw = rawstars->data[match->raw]; - pmAstromObj *ref = refstars->data[match->ref]; + pmAstromObj *raw = rawstars->data[match->raw]; + pmAstromObj *ref = refstars->data[match->ref]; - psMetadata *row = psMetadataAlloc (); - psMetadataAdd (row, PS_LIST_TAIL, "RA", PS_DATA_F64, "right ascension (deg, J2000)", PM_DEG_RAD*ref->sky->r); - psMetadataAdd (row, PS_LIST_TAIL, "DEC", PS_DATA_F64, "declination (deg, J2000)", PM_DEG_RAD*ref->sky->d); - psMetadataAdd (row, PS_LIST_TAIL, "X_CHIP", PS_DATA_F32, "x coord on chip", raw->chip->x); - psMetadataAdd (row, PS_LIST_TAIL, "Y_CHIP", PS_DATA_F32, "y coord on chip", raw->chip->y); - psMetadataAdd (row, PS_LIST_TAIL, "X_FPA", PS_DATA_F32, "x coord on focal plane", raw->FP->x); - psMetadataAdd (row, PS_LIST_TAIL, "Y_FPA", PS_DATA_F32, "y coord on focal plane", raw->FP->y); - psMetadataAdd (row, PS_LIST_TAIL, "MAG_INST", PS_DATA_F32, "instrumental magnitude", raw->Mag); - psMetadataAdd (row, PS_LIST_TAIL, "MAG_REF", PS_DATA_F32, "reference star magnitude", ref->Mag); - psMetadataAdd (row, PS_LIST_TAIL, "COLOR_REF",PS_DATA_F32, "reference star color", ref->Color); - psMetadataAdd (row, PS_LIST_TAIL, "CHIP_ID", PS_DATA_STRING, "chip identifier", chipName); - // XXX need to add the reference color, but this needs getstar / dvo.photcodes for the reference to be refined. + psMetadata *row = psMetadataAlloc (); + psMetadataAdd (row, PS_LIST_TAIL, "RA", PS_DATA_F64, "right ascension (deg, J2000)", PM_DEG_RAD*ref->sky->r); + psMetadataAdd (row, PS_LIST_TAIL, "DEC", PS_DATA_F64, "declination (deg, J2000)", PM_DEG_RAD*ref->sky->d); + psMetadataAdd (row, PS_LIST_TAIL, "X_CHIP", PS_DATA_F32, "x coord on chip", raw->chip->x); + psMetadataAdd (row, PS_LIST_TAIL, "Y_CHIP", PS_DATA_F32, "y coord on chip", raw->chip->y); + psMetadataAdd (row, PS_LIST_TAIL, "X_FPA", PS_DATA_F32, "x coord on focal plane", raw->FP->x); + psMetadataAdd (row, PS_LIST_TAIL, "Y_FPA", PS_DATA_F32, "y coord on focal plane", raw->FP->y); + psMetadataAdd (row, PS_LIST_TAIL, "MAG_INST", PS_DATA_F32, "instrumental magnitude", raw->Mag); + psMetadataAdd (row, PS_LIST_TAIL, "MAG_REF", PS_DATA_F32, "reference star magnitude", ref->Mag); + psMetadataAdd (row, PS_LIST_TAIL, "COLOR_REF",PS_DATA_F32, "reference star color", ref->Color); + psMetadataAdd (row, PS_LIST_TAIL, "CHIP_ID", PS_DATA_STRING, "chip identifier", chipName); + // XXX need to add the reference color, but this needs getstar / dvo.photcodes for the reference to be refined. - psArrayAdd (table, 100, row); - psFree (row); - } - } - } + psArrayAdd (table, 100, row); + psFree (row); + } + } + } } psFree (view); if (table->n == 0) { - psFree(table); - return true; + psFree(table); + return true; } Index: branches/eam_branches/20090522/psModules/src/objects/pmSourceMatch.c =================================================================== --- branches/eam_branches/20090522/psModules/src/objects/pmSourceMatch.c (revision 24529) +++ branches/eam_branches/20090522/psModules/src/objects/pmSourceMatch.c (revision 24557) @@ -45,4 +45,5 @@ psVector **x, psVector **y, // Coordinate vectors to return psVector **mag, psVector **magErr, // Magnitude and error vectors to return + psVector **indices, // Indices for sources const psArray *sources // Input sources ) @@ -58,13 +59,13 @@ *mag = psVectorRecycle(*mag, numSources, PS_TYPE_F32); *magErr = psVectorRecycle(*magErr, numSources, PS_TYPE_F32); + *indices = psVectorRecycle(*indices, numSources, PS_TYPE_S32); float xMin = INFINITY, xMax = -INFINITY, yMin = INFINITY, yMax = -INFINITY; // Bounds of sources long num = 0; // Number of valid sources for (long i = 0; i < numSources; i++) { pmSource *source = sources->data[i]; // Source of interest - if (!source) continue; - if (source->mode & SOURCE_MASK) continue; - if (!isfinite(source->psfMag)) continue; - if (!isfinite(source->errMag)) continue; - if (source->psfMag > SOURCE_FAINTEST) continue; + if (!source || (source->mode & SOURCE_MASK) || !isfinite(source->psfMag) || + !isfinite(source->errMag) || source->psfMag > SOURCE_FAINTEST) { + continue; + } float xSrc, ySrc; // Coordinates of source @@ -78,5 +79,6 @@ (*y)->data.F32[num] = ySrc; (*mag)->data.F32[num] = source->psfMag; - (*magErr)->data.F32[num] = source->errMag ; + (*magErr)->data.F32[num] = source->errMag; + (*indices)->data.S32[num] = i; num++; } @@ -85,4 +87,5 @@ (*mag)->n = num; (*magErr)->n = num; + (*indices)->n = num; if (*bounds) { @@ -175,6 +178,7 @@ psVector *xImage = NULL, *yImage = NULL; // Coordinates of sources psVector *magImage = NULL, *magErrImage = NULL; // Magnitude and mag - - int numSources = sourcesParse(&boundsImage, &xImage, &yImage, &magImage, &magErrImage, + psVector *indices = NULL; // Indices for sources + + int numSources = sourcesParse(&boundsImage, &xImage, &yImage, &magImage, &magErrImage, &indices, sources); // Number of sources @@ -187,5 +191,6 @@ for (int j = 0; j < numSources; j++) { pmSourceMatch *match = pmSourceMatchAlloc(); // Match data - pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], i, j); + pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], + i, indices->data.S32[j]); matches->data[j] = match; } @@ -206,5 +211,6 @@ for (int j = 0, k = numMaster; j < numSources; j++, k++) { pmSourceMatch *match = pmSourceMatchAlloc(); // Match data - pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], i, j); + pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], + i, indices->data.S32[j]); matches->data[k] = match; } @@ -231,10 +237,12 @@ // Record the match pmSourceMatch *match = matches->data[index]; // Match data - pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], i, j); + pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], + i, indices->data.S32[j]); numMatch++; } else { // Add to the master list pmSourceMatch *match = pmSourceMatchAlloc(); // Match data - pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], i, j); + pmSourceMatchAdd(match, magImage->data.F32[j], magErrImage->data.F32[j], + i, indices->data.S32[j]); xMaster->data.F32[numMaster] = xImage->data.F32[j]; yMaster->data.F32[numMaster] = yImage->data.F32[j]; Index: branches/eam_branches/20090522/psconfig/pscheckperl =================================================================== --- branches/eam_branches/20090522/psconfig/pscheckperl (revision 24529) +++ branches/eam_branches/20090522/psconfig/pscheckperl (revision 24557) @@ -78,10 +78,14 @@ if ($modver eq "") { $modver = 0; } system ("pscheckmods $module $modver"); - if ($? == 0) { + if (($? == 0) and not defined $force{$module} and not defined $force{'all'}) { # print "$module: found\n"; next; } - print "$module: missing\n"; + if (defined $force{$module}) { + print "$module: force\n"; + } else { + print "$module: missing\n"; + } unless ($build or defined $force{$module} or defined $force{'all'}) { $Nmissing ++; Index: branches/eam_branches/20090522/psconfig/tagsets/ipp-2.8.dist =================================================================== --- branches/eam_branches/20090522/psconfig/tagsets/ipp-2.8.dist (revision 24529) +++ branches/eam_branches/20090522/psconfig/tagsets/ipp-2.8.dist (revision 24557) @@ -69,4 +69,6 @@ YYYYY DataStore ipp-2-8 -0 + YYYYY ppMops ipp-2-8 -0 + YNNNN extsrc/gpcsw ipp-2-8 -0 Index: branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.dist =================================================================== --- branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.dist (revision 24557) +++ branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.dist (revision 24557) @@ -0,0 +1,76 @@ +# necessary fields: +# |-- tag? +# ||-- build? +# |||-- package? +# ||||-- update directory from CVS? +# |||||-- build for developer? +# ||||| +# ||||| module branch name tag version +# ||||| + YYNNY Ohana ipp-2-9 -0 + NNYYN ohana.base ipp-2-9 -0 + NNYYN libohana ipp-2-9 -0 + NNYYN libfits ipp-2-9 -0 + NNYYN libautocode ipp-2-9 -0 + NNYYN libdvo ipp-2-9 -0 + NNYYN libkapa ipp-2-9 -0 + NNYYN libtap.ohana ipp-2-9 -0 + NNYYN addstar ipp-2-9 -0 + NNYYN delstar ipp-2-9 -0 + NNYYN getstar ipp-2-9 -0 + NNYYN ohana.tools ipp-2-9 -0 + NNYYN kapa2 ipp-2-9 -0 + NNYYN relphot ipp-2-9 -0 + NNYYN relastro ipp-2-9 -0 + NNYYN uniphot ipp-2-9 -0 + NNYYN opihi.base ipp-2-9 -0 + NNYYN mana ipp-2-9 -0 + NNYYN dvo ipp-2-9 -0 + NNYYN pantasks ipp-2-9 -0 + NNYYN pcontrol ipp-2-9 -0 + NNYYN pclient ipp-2-9 -0 + + YNNYY Nebulous/nebclient ipp-2-9 -0 + YNNYY Nebulous ipp-2-9 -0 + YNNYY Nebulous-Server ipp-2-9 -0 + YYYYY PS-IPP-Metadata-Config ipp-2-9 -0 + YYYYY PS-IPP-Config ipp-2-9 -0 + + YYYYY psLib ipp-2-9 -0 + YYYYY psModules ipp-2-9 -0 + YYYYY ppStats ipp-2-9 -0 + YYYYY psphot ipp-2-9 -0 + YYYYY psastro ipp-2-9 -0 + YYYYY ppConfigDump ipp-2-9 -0 + YYYYY ppImage ipp-2-9 -0 + YYYYY ppNoiseMap ipp-2-9 -0 + YYYYY ppNorm ipp-2-9 -0 + YYYYY ppMerge ipp-2-9 -0 + YNNYN pedestal ipp-2-9 -0 + YYYYY dvoTools ipp-2-9 -0 + YYYYY pswarp ipp-2-9 -0 + YYYYY ppArith ipp-2-9 -0 + YYYYY ppStack ipp-2-9 -0 + YYYYY ppSub ipp-2-9 -0 + YYYYY ppSim ipp-2-9 -0 + + YNNYY glueforge ipp-2-9 -0 + YNNYY dbconfig ipp-2-9 -0 + NNNNY ippdb.src + YYYNN ippdb ipp-2-9 -0 + YYYYY PS-IPP-PStamp ipp-2-9 -0 + YYYYY pstamp ipp-2-9 -0 + YYYYY ippTools ipp-2-9 -0 + YYYYY ippScripts ipp-2-9 -0 + YYYYY ippTasks ipp-2-9 -0 + + YYYYY ippconfig ipp-2-9 -0 + YNYYN psconfig ipp-2-9 -0 + YNYYN ippMonitor ipp-2-9 -0 + YYYYY DataStore ipp-2-9 -0 + + YYYYY ppMops ipp-2-9 -0 + + NNNNN extsrc/gpcsw ipp-2-9 -0 + +# there are externally required C libraries and perl modules (see INSTALL) Index: branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.libs =================================================================== --- branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.libs (revision 24557) +++ branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.libs (revision 24557) @@ -0,0 +1,97 @@ +# this file defines C libraries and C headers needed by the ipp, and information on finding them + +# directories to search: /lib, /usr/lib, /usr/X11R6/lib, etc. +# modifications based on the architecture (eg, PATH/lib64, etc) +# additional locations based on the env variables +# dlltype to use + +# each entry contains: +# type : lib / include +# name +# alternate names +# alternate paths +# tarball name +# tar directory +# force install by default? +# configure options +# make options +# make install options + + +# Build tools +bin autoconf NONE NONE autoconf-2.63.tar.gz autoconf-2.63 N NONE NONE NONE +bin automake NONE NONE automake-1.10.tar.gz automake-1.10 N NONE NONE NONE +bin libtool NONE NONE libtool-2.2.6a.tar.gz libtool-2.2.6 N NONE NONE NONE +bin pkg-config NONE NONE pkg-config-0.23.tar.gz pkg-config-0.23 N NONE NONE NONE + +lib libm NONE NONE NONE NONE N NONE NONE NONE +lib libX11 NONE NONE NONE NONE N NONE NONE NONE +lib libpthread NONE NONE NONE NONE N NONE NONE NONE +lib libncurses curses,termcap NONE ncurses-5.6.tar.gz ncurses-5.6 N NONE NONE NONE +lib libreadline NONE NONE readline-5.2-p12.tar.gz readline-5.2-p12 Y NONE NONE NONE +lib libz NONE NONE zlib-1.2.3.tar.gz zlib-1.2.3 N --shared NONE NONE +lib libpng NONE NONE libpng-1.2.15.tar.gz libpng-1.2.15 N NONE NONE NONE +lib libjpeg NONE jpeg jpegsrc.v6b-p1.tar.gz jpeg-6b N --enable-shared NONE install-lib +lib libcfitsio NONE NONE cfitsio3100-p2.tar.gz cfitsio3100-p2 N --enable-shared shared NONE +lib libmysqlclient NONE mysql mysql-5.0.51a.tar.gz mysql-5.0.51a N NONE NONE NONE +lib libgsl NONE NONE gsl-1.11.tar.gz gsl-1.11 N NONE NONE NONE +lib libfftw3f NONE NONE fftw-3.0.1.tar.gz fftw-3.0.1 N --enable-float,--enable-shared,--disable-fortran NONE NONE + + +inc X11/Xatom.h NONE NONE NONE NONE N NONE NONE NONE +inc X11/Xlib.h NONE NONE NONE NONE N NONE NONE NONE +inc X11/Xresource.h NONE NONE NONE NONE N NONE NONE NONE +inc X11/Xutil.h NONE NONE NONE NONE N NONE NONE NONE +inc X11/cursorfont.h NONE NONE NONE NONE N NONE NONE NONE +inc X11/keysym.h NONE NONE NONE NONE N NONE NONE NONE +inc X11/keysymdef.h NONE NONE NONE NONE N NONE NONE NONE +inc arpa/inet.h NONE NONE NONE NONE N NONE NONE NONE +inc assert.h NONE NONE NONE NONE N NONE NONE NONE +inc complex.h NONE NONE NONE NONE N NONE NONE NONE +inc ctype.h NONE NONE NONE NONE N NONE NONE NONE +inc errno.h NONE NONE NONE NONE N NONE NONE NONE +inc fcntl.h NONE NONE NONE NONE N NONE NONE NONE +inc fftw3.h NONE NONE fftw-3.0.1.tar.gz fftw-3.0.1 N --enable-float,--enable-shared,--disable-fortran NONE NONE +inc fitsio.h NONE NONE cfitsio3100-pap.tar.gz cfitsio3100-pap N --enable-shared shared NONE +inc glob.h NONE NONE NONE NONE N NONE NONE NONE +inc gsl/gsl_randist.h NONE NONE gsl-1.11.tar.gz gsl-1.11 N NONE NONE NONE +inc gsl/gsl_rng.h NONE NONE gsl-1.11.tar.gz gsl-1.11 N NONE NONE NONE +inc inttypes.h NONE NONE NONE NONE N NONE NONE NONE +inc jpeglib.h NONE jpeg jpegsrc.v6b-p1.tar.gz jpeg-6b N --enable-shared NONE install-lib +inc limits.h NONE NONE NONE NONE N NONE NONE NONE +inc malloc.h NONE NONE NONE NONE N NONE NONE NONE +inc math.h NONE NONE NONE NONE N NONE NONE NONE +inc memory.h NONE NONE NONE NONE N NONE NONE NONE +inc mysql.h NONE mysql mysql-5.0.51a.tar.gz mysql-5.0.51a N NONE NONE NONE +inc netdb.h NONE NONE NONE NONE N NONE NONE NONE +inc netinet/ip.h NONE NONE NONE NONE N NONE NONE NONE +inc png.h NONE NONE libpng-1.2.15.tar.gz libpng-1.2.15 N NONE NONE NONE +inc pthread.h NONE NONE NONE NONE N NONE NONE NONE +inc readline/history.h NONE NONE readline-5.2-p12.tar.gz readline-5.2-p12 N NONE NONE NONE +inc readline/readline.h NONE NONE readline-5.2-p12.tar.gz readline-5.2-p12 N NONE NONE NONE +inc regex.h NONE NONE NONE NONE N NONE NONE NONE +inc signal.h NONE NONE NONE NONE N NONE NONE NONE +inc stdint.h NONE NONE NONE NONE N NONE NONE NONE +inc stdio.h NONE NONE NONE NONE N NONE NONE NONE +inc stdlib.h NONE NONE NONE NONE N NONE NONE NONE +inc string.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/ipc.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/resource.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/sem.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/socket.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/stat.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/time.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/types.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/uio.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/un.h NONE NONE NONE NONE N NONE NONE NONE +inc sys/wait.h NONE NONE NONE NONE N NONE NONE NONE +inc time.h NONE NONE NONE NONE N NONE NONE NONE +inc unistd.h NONE NONE NONE NONE N NONE NONE NONE +inc zlib.h NONE NONE zlib-1.2.3.tar.gz zlib-1.2.3 N --shared NONE NONE + +# xml is currently not used by IPP +# lib xml2 NONE NONE NONE NONE Y NONE NONE NONE + +# doxygen is having some unknown build issues on alala +# bin doxygen NONE NONE doxygen-1.5.1.src.tar.gz doxygen-1.5.1 N NONE NONE NONE + Index: branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.perl =================================================================== --- branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.perl (revision 24557) +++ branches/eam_branches/20090522/psconfig/tagsets/ipp-2.9.perl (revision 24557) @@ -0,0 +1,83 @@ +# NN Name Tarball Version Optional Responses + 00 Getopt::Long Getopt-Long-2.36.tar.gz 2.3 n + 00 Module::Build Module-Build-0.2806.tar.gz 0.2806 + 01 ExtUtils::MakeMaker ExtUtils-MakeMaker-6.31.tar.gz 0 + 02 Params::Validate Params-Validate-0.91.tar.gz 0.77 +# 02 Apache::Test Apache-Test-1.29.tar.gz 1.29 + 03 DateTime::TimeZone DateTime-TimeZone-0.59.tar.gz 0 + 04 DateTime::Locale DateTime-Locale-0.33.tar.gz 0 + 05 Time::Local Time-Local-1.17.tar.gz 0 + 06 DateTime DateTime-0.36.tar.gz 0 + 07 MIME::Base64 MIME-Base64-3.07.tar.gz 0 + 08 IO::Compress::Base IO-Compress-Base-2.003.tar.gz 0 + 09 Compress::Raw::Zlib Compress-Raw-Zlib-2.003.tar.gz 0 + 10 Class::Factory::Util Class-Factory-Util-1.6.tar.gz 0 + 11 DateTime::Format::Strptime DateTime-Format-Strptime-1.0700.tar.gz 0 + 12 Net::Domain::TLD Net-Domain-TLD-1.65.tar.gz 0 + 13 Sub::Uplevel Sub-Uplevel-0.14.tar.gz 0 + 14 HTML::Tagset HTML-Tagset-3.10.tar.gz 0 + 15 Digest Digest-1.15.tar.gz 0 + 16 IO::Compress::Zlib::Extra IO-Compress-Zlib-2.003.tar.gz 0 + 17 version version-0.70.tar.gz 0 + 18 Text::Balanced Text-Balanced-v2.0.0.tar.gz 0 + 19 DateTime::Format::Builder DateTime-Format-Builder-0.7807.tar.gz 0 + 20 ExtUtils::Manifest ExtUtils-Manifest-1.51.tar.gz 0 + 21 URI URI-1.35.tar.gz 1.30 + 22 Data::Validate::Domain Data-Validate-Domain-0.05.tar.gz 0 + 23 Test::Exception Test-Exception-0.24.tar.gz 0 + 24 Tree::DAG_Node Tree-DAG_Node-1.05.tar.gz 0 + 25 Array::Compare Array-Compare-1.13.tar.gz 0 + 26 HTML::Parser HTML-Parser-3.56.tar.gz 0 + 27 Digest::MD5 Digest-MD5-2.36.tar.gz 0 + 28 Net::FTP libnet-1.19.tar.gz 0 + 29 Compress::Zlib Compress-Zlib-2.003.tar.gz 0 + 30 Locale::Maketext::Simple Locale-Maketext-Simple-0.18.tar.gz 0 + 31 Parse::RecDescent Parse-RecDescent-1.94.tar.gz 1.94 + 32 Class::Accessor Class-Accessor-0.30.tar.gz 0.19 + 33 DateTime::Format::ISO8601 DateTime-Format-ISO8601-0.06.tar.gz 0.06 + 34 CGI CGI.pm-3.25.tar.gz 3 + 35 Test::Cmd Test-Cmd-1.05.tar.gz 1.05 + 36 Net::HTTPServer Net-HTTPServer-1.1.1.tar.gz 1.1.1 + 37 LWP libwww-perl-5.805.tar.gz 0 + 38 Digest::MD5::File Digest-MD5-File-0.05.tar.gz 0.03 + 39 File::Temp File-Temp-0.18.tar.gz 0.16 + 40 Data::Validate::URI Data-Validate-URI-0.01.tar.gz 0.01 + 41 Test::Warn Test-Warn-0.08.tar.gz 0 + 42 YAML YAML-0.62.tar.gz 0.58 y + 43 Module::Load Module-Load-0.10.tar.gz 0 + 44 Params::Check Params-Check-0.25.tar.gz 0 + 45 Template Template-Toolkit-2.16.tar.gz 0 n,n + 46 Statistics::Descriptive Statistics-Descriptive-2.6.tar.gz 2.6 + 47 Storable Storable-2.15.tar.gz 0 + 48 IO::String IO-String-1.08.tar.gz 0 + 49 Date::Parse TimeDate-1.16.tar.gz 0 + 50 Digest::SHA1 Digest-SHA1-2.11.tar.gz 0 + 51 DB_File DB_File-1.814.tar.gz 0 + 52 File::NFSLock File-NFSLock-1.20.tar.gz 0 + 53 Heap Heap-0.71.tar.gz 0 + 54 Module::Load::Conditional Module-Load-Conditional-0.16.tar.gz 0 + 55 IPC::Run IPC-Run-0.80.tar.gz 0 + 56 Cache Cache-2.04.tar.gz 0 + 57 IPC::Cmd IPC-Cmd-0.36.tar.gz 0.36 + 58 SOAP::Lite SOAP-Lite-0.69.tar.gz 0 yes,yes,no + 59 Log::Log4perl Log-Log4perl-1.10.tar.gz 0 +# 60 File::ExtAttr File-ExtAttr-1.04.tar.gz 0 + 61 Text::Glob Text-Glob-0.08.tar.gz 0.08 + 62 Number::Compare Number-Compare-0.01.tar.gz 0.01 + 63 File::Find::Rule File-Find-Rule-0.30.tar.gz 0.30 + 64 Astro::FITS::CFITSIO Astro-FITS-CFITSIO-1.05.tar.gz 0 + 65 Test::More Test-Simple-0.74.tar.gz 0.49 +# 66 Apache::DBI Apache-DBI-1.06.tar.gz 0 +# 67 Apache2::SOAP Apache2-SOAP-0.72.tar.gz 0 + 68 Test::URI Test-URI-1.08.tar.gz 0 +# 69 Sys::Statistics::Linux::DiskUsage Sys-Statistics-Linux-0.26.tar.gz 0 +# 70 Config::YAML Config-YAML-1.42.tar.gz 0 +# 72 File::ExtAttr File-ExtAttr-1.07.tar.gz 0 + 73 DBI DBI-1.601.tar.gz 0 + 71 DBD::mysql DBD-mysql-4.006.tar.gz 0 +# 74 Net::Server::Daemonize Net-Server-0.97.tar.gz 0.05 + 75 File::Path File-Path-2.04.tar.gz + 76 File::Mountpoint File-Mountpoint-0.01.tar.gz 0.01 + 77 Filesys::Df Filesys-Df-0.92.tar.gz 0.92 + 78 SQL::Interp SQL-Interp-1.06.tar.gz + 79 Log::Dispatch::Email::MailSend Log-Dispatch-2.22.tar.gz Index: branches/eam_branches/20090522/psphot/src/psphotFindFootprints.c =================================================================== --- branches/eam_branches/20090522/psphot/src/psphotFindFootprints.c (revision 24529) +++ branches/eam_branches/20090522/psphot/src/psphotFindFootprints.c (revision 24557) @@ -59,5 +59,5 @@ psArray *mergedFootprints = pmFootprintArraysMerge(detections->footprints, footprints, includePeaks); - psLogMsg ("psphot", PS_LOG_MINUTIA, "merged %ld new footprints with %ld existing ones: %f sec\n", footprints->n, detections->footprints->n, psTimerMark ("psphot.footprints")); + psLogMsg ("psphot", PS_LOG_MINUTIA, "merged %ld new footprints with %ld existing ones: %f sec\n", footprints->n, (detections->footprints ? detections->footprints->n : 0), psTimerMark ("psphot.footprints")); psFree(footprints); Index: branches/eam_branches/20090522/psphot/src/psphotMaskReadout.c =================================================================== --- branches/eam_branches/20090522/psphot/src/psphotMaskReadout.c (revision 24529) +++ branches/eam_branches/20090522/psphot/src/psphotMaskReadout.c (revision 24557) @@ -27,5 +27,5 @@ } if (!readout->variance) { - if (!pmReadoutGenerateVariance(readout, true)) { + if (!pmReadoutGenerateVariance(readout, NULL, true)) { psError (PSPHOT_ERR_CONFIG, false, "trouble creating variance"); return false; Index: branches/eam_branches/20090522/psphot/src/psphotReadoutMinimal.c =================================================================== --- branches/eam_branches/20090522/psphot/src/psphotReadoutMinimal.c (revision 24529) +++ branches/eam_branches/20090522/psphot/src/psphotReadoutMinimal.c (revision 24557) @@ -46,5 +46,6 @@ // find the detections (by peak and/or footprint) in the image. - pmDetections *detections = psphotFindDetections (NULL, readout, recipe); + pmDetections *detections = pmDetectionsAlloc(); // New detections; allocated to ensure pass=2 + detections = psphotFindDetections(detections, readout, recipe); if (!detections) { psLogMsg ("psphot", 3, "unable to find detections in this image"); Index: branches/eam_branches/20090522/pswarp/src/pswarpLoop.c =================================================================== --- branches/eam_branches/20090522/pswarp/src/pswarpLoop.c (revision 24529) +++ branches/eam_branches/20090522/pswarp/src/pswarpLoop.c (revision 24557) @@ -79,5 +79,5 @@ "SKYCELL.CAMERA"); ///< Name of camera for skycell pmConfigCamerasCull(config, skyCamera); - pmConfigRecipesCull(config, "PSWARP,PPSTATS,PSPHOT,MASKS,JPEG"); + pmConfigRecipesCull(config, "PSWARP,PPSTATS,PSPHOT,PSASTRO,MASKS,JPEG"); @@ -297,4 +297,22 @@ } + // Correct image for change in the plate scale + { + psAssert(input && input->fpa && input->fpa->toSky, "Require astrometry for input"); + psAssert(outFPA && outFPA && outFPA->toSky, "Require astrometry for output"); + + double inScale = input->fpa->toSky->Xs + input->fpa->toSky->Ys; // Plate scale for input + double outScale = outFPA->toSky->Xs + outFPA->toSky->Ys; // Plate scale for output + float correction = PS_SQR(outScale / inScale); // Correction factor to apply to image + psLogMsg("pswarp", PS_LOG_INFO, "Correcting flux by %f to account for pixel scales", correction); + + psBinaryOp(output->image, output->image, "*", psScalarAlloc(correction, PS_TYPE_F32)); + if (output->variance) { + psBinaryOp(output->variance, output->variance, "*", + psScalarAlloc(PS_SQR(correction), PS_TYPE_F32)); + } + } + + // Set variance factor { Index: branches/eam_branches/20090522/pswarp/src/pswarpTransformReadout.c =================================================================== --- branches/eam_branches/20090522/pswarp/src/pswarpTransformReadout.c (revision 24529) +++ branches/eam_branches/20090522/pswarp/src/pswarpTransformReadout.c (revision 24557) @@ -109,6 +109,6 @@ // create jobs and supply them to the threads - for (int gridY = yGridMin; gridY < yGridMax; gridY++) { - for (int gridX = xGridMin; gridX < xGridMax; gridX++) { + for (int gridY = yGridMin; gridY <= yGridMax; gridY++) { + for (int gridX = xGridMin; gridX <= xGridMax; gridX++) { pswarpTransformTileArgs *args = pswarpTransformTileArgsAlloc(); args->input = psMemIncrRefCounter(input); @@ -171,5 +171,4 @@ psFree(job); } - psFree(interp); psFree(grid); @@ -202,4 +201,5 @@ } } + psFree(interp); if (goodPixels > 0) { Index: branches/eam_branches/20090522/tools/ipp_apply_burntool.pl =================================================================== --- branches/eam_branches/20090522/tools/ipp_apply_burntool.pl (revision 24529) +++ branches/eam_branches/20090522/tools/ipp_apply_burntool.pl (revision 24557) @@ -111,5 +111,4 @@ foreach my $file (@files) { my $exp_id = $file->{exp_id}; - if ($skip_burned and ($file->{user_1} > 0.5)) { next; } my $rawImfile = $file->{uri}; @@ -147,30 +146,31 @@ } - # uncompress the image (do we need to check if it is compressed?) - my $status = vsystem ("$funpack -S $rawImfileReal > $tmpImfileReal", $REALRUN); - if ($status) { - &my_die("failed on funpack"); - } - - if ($RAWTABLES) { - $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN); - } else { - $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN); - } - if ($status) { - &my_die("failed on burntool"); - } - - # compress the image (do we need to check if it is compressed?) - $status = vsystem ("$fpack -S $tmpImfileReal > $outImfileReal", $REALRUN); - if ($status) { - &my_die("failed on fpack"); - } - - $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 1.0", 1); - if ($status) { - &my_die("failed to update imfile"); - } - + if (! ($skip_burned and ($file->{user_1} > 0.5))) { + # uncompress the image (do we need to check if it is compressed?) + my $status = vsystem ("$funpack -S $rawImfileReal > $tmpImfileReal", $REALRUN); + if ($status) { + &my_die("failed on funpack"); + } + + if ($RAWTABLES) { + $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN); + } else { + $status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN); + } + if ($status) { + &my_die("failed on burntool"); + } + + # compress the image (do we need to check if it is compressed?) + $status = vsystem ("$fpack -S $tmpImfileReal > $outImfileReal", $REALRUN); + if ($status) { + &my_die("failed on fpack"); + } + + $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 1.0", 1); + if ($status) { + &my_die("failed to update imfile"); + } + } # save the artifact file for the next image if ($RAWTABLES) {