Index: branches/meh_branches/ppstack_test/Nebulous-Server/Build.PL =================================================================== --- branches/meh_branches/ppstack_test/Nebulous-Server/Build.PL (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous-Server/Build.PL (revision 33415) @@ -49,4 +49,5 @@ bin/neb-voladd bin/neb-voladm + bin/neb-insedit bin/neb-host bin/nebdiskd Index: branches/meh_branches/ppstack_test/Nebulous-Server/MANIFEST =================================================================== --- branches/meh_branches/ppstack_test/Nebulous-Server/MANIFEST (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous-Server/MANIFEST (revision 33415) @@ -10,4 +10,5 @@ bin/neb-cabadd bin/neb-voladm +bin/neb-insedit bin/neb-host bin/nebdiskd Index: branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-admin =================================================================== --- branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-admin (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-admin (revision 33415) @@ -29,4 +29,5 @@ $pending, $balance, + $retarget, $balance_N_sources, $balance_M_destinations, @@ -49,4 +50,5 @@ 'pendingreplicate|r' => \$pending, 'pendingbalance|b' => \$balance, + 'pendingtarget|t' => \$retarget, 'balancesources|N=s' => \$balance_N_sources, 'balancedestinations|M=s' => \$balance_M_destinations, @@ -64,7 +66,7 @@ pod2usage( -msg => "--limit is meaningless without --pendingreplicate", -exitval => 2 ) - if defined $limit and not (defined $pending or defined $balance); + if defined $limit and not (defined $pending or defined $balance or defined $retarget); pod2usage( -msg => "no operation specified", -exitval => 2 ) - unless defined $pending or defined $balance; + unless defined $pending or defined $balance or defined $retarget; pod2usage( -msg => "--pendingbalance needs --balancesources and --balancedestinations options", -exitval => 2) @@ -81,4 +83,7 @@ elsif (defined $balance) { $pidfile = '/var/tmp/neb-admin.balance'; +} +elsif (defined $retarget) { + $pidfile = '/var/tmp/neb-admin.retarget'; } else { @@ -107,4 +112,6 @@ } elsif ($balance) { balance(); +} elsif ($retarget) { + retarget(); } else { die "THIS SHOULD NOT HAPPEN"; @@ -306,5 +313,5 @@ SELECT S.*,vol_id FROM ( -- Select objects that are in our so_id range, with two copies (and two requested copies) - SELECT MAX(ins_id) AS MAXins_id,MIN(ins_id) AS MINins_id,so_id,ext_id,value AS user_copies FROM + SELECT MAX(ins_id) AS MAXins_id,MIN(ins_id) AS MINins_id,so_id,ext_id FROM storage_object JOIN storage_object_xattr USING(so_id) RIGHT JOIN instance USING(so_id) WHERE name = 'user.copies' AND value >= 2 @@ -462,5 +469,5 @@ my $so_id = $obj->{so_id}; my $key = $obj->{ext_id}; - my $copies = $obj->{user_copies}; +# my $copies = $obj->{user_copies}; my $source_vol_id = $obj->{vol_id}; @@ -480,5 +487,5 @@ my %md = ( key => $key, - copies => $copies, +# copies => $copies, so_id => $so_id, source_vol_id => $source_vol_id, @@ -524,4 +531,218 @@ } +sub retarget +{ + my %destination; + my %vol_id_list = ('ipp004.0' => 1,'ipp005.0' => 2, 'ipp006.0' => 3, 'ipp007.0' => 4,'ipp008.0' => 50,'ipp009.0' => 6,'ipp010.0' => 7, + 'ipp011.0' => 8,'ipp012.0' => 15,'ipp013.0' => 16, 'ipp014.0' => 18,'ipp015.0' => 17,'ipp016.0' => 9,'ipp017.0' => 10,'ipp018.0' => 11,'ipp019.0' => 12, + 'ipp020.0' => 52,'ipp021.0' => 14,'ipp023.0' => 19,'ipp024.0' => 20,'ipp025.0' => 32,'ipp026.0' => 21, 'ipp027.0' => 49, + 'ipp028.0' => 23,'ipp029.0' => 24,'ipp030.0' => 25,'ipp031.0' => 26,'ipp032.0' => 27,'ipp033.0' => 28,'ipp034.0' => 29, + 'ipp035.0' => 30,'ipp036.0' => 31,'ipp037.0' => 51,'ipp038.0' => 33,'ipp039.0' => 34,'ipp040.0' => 35,'ipp041.0' => 36, + 'ipp042.0' => 37,'ipp043.0' => 38,'ipp044.0' => 39,'ipp045.0' => 40,'ipp046.0' => 41,'ipp047.0' => 42,'ipp048.0' => 43, + 'ipp049.0' => 44,'ipp050.0' => 45,'ipp051.0' => 46,'ipp052.0' => 47,'ipp053.0' => 48,'ipp054.0' => 62,'ipp055.0' => 63, + 'ipp056.0' => 64,'ipp057.0' => 65,'ipp058.0' => 66,'ipp059.0' => 67,'ipp060.0' => 68,'ipp061.0' => 69,'ipp062.0' => 70, + 'ipp063.0' => 71,'ipp064.0' => 72,'ipp065.0' => 73,'ipp066.0' => 74); + %{ $destination{VOL} } = ('ota01' => 7, 'ota02' => 8, 'ota03' => 3, 'ota04' => 3, 'ota05' => 4, 'ota06' => 4, + 'ota10' => 50, 'ota11' => 50, 'ota12' => 6, 'ota13' => 6, 'ota14' => 7, 'ota15' => 8, 'ota16' => 15, 'ota17' => 15, + 'ota20' => 16, 'ota21' => 18, 'ota22' => 18, 'ota23' => 17, 'ota24' => 9, 'ota25' => 9, 'ota26' => 10, 'ota27' => 11, + 'ota30' => 12, 'ota31' => 52, 'ota32' => 14, 'ota33' => 14, 'ota34' => 19, 'ota35' => 16, 'ota36' => 20, 'ota37' => 12, + 'ota40' => 32, 'ota41' => 11, 'ota42' => 10, 'ota43' => 17, 'ota44' => 49, 'ota45' => 23, 'ota46' => 24, 'ota47' => 25, + 'ota50' => 26, 'ota51' => 27, 'ota52' => 28, 'ota53' => 29, 'ota54' => 30, 'ota55' => 31, 'ota56' => 51, 'ota57' => 33, + 'ota60' => 34, 'ota61' => 35, 'ota62' => 36, 'ota63' => 37, 'ota64' => 38, 'ota65' => 39, 'ota66' => 41, 'ota67' => 42, + 'ota71' => 43, 'ota72' => 44, 'ota73' => 45, 'ota74' => 46, 'ota75' => 47, 'ota76' => 48); + %{ $destination{HOST} } = ('ota01' => 'ipp010','ota02' => 'ipp011','ota03' => 'ipp006','ota04' => 'ipp054','ota05' => 'ipp007','ota06' => 'ipp055', + 'ota10' => 'ipp008','ota11' => 'ipp056','ota12' => 'ipp009','ota13' => 'ipp057','ota14' => 'ipp058','ota15' => 'ipp059','ota16' => 'ipp012','ota17' => 'ipp060', + 'ota20' => 'ipp013','ota21' => 'ipp014','ota22' => 'ipp061','ota23' => 'ipp015','ota24' => 'ipp016','ota25' => 'ipp062','ota26' => 'ipp064','ota27' => 'ipp065', + 'ota30' => 'ipp066','ota31' => 'ipp020','ota32' => 'ipp063','ota33' => 'ipp021','ota34' => 'ipp023','ota35' => 'ipp013','ota36' => 'ipp024','ota37' => 'ipp019', + 'ota40' => 'ipp025','ota41' => 'ipp018','ota42' => 'ipp017','ota43' => 'ipp015','ota44' => 'ipp027','ota45' => 'ipp028','ota46' => 'ipp029','ota47' => 'ipp030', + 'ota50' => 'ipp031','ota51' => 'ipp032','ota52' => 'ipp033','ota53' => 'ipp034','ota54' => 'ipp035','ota55' => 'ipp036','ota56' => 'ipp037','ota57' => 'ipp038', + 'ota60' => 'ipp039','ota61' => 'ipp040','ota62' => 'ipp041','ota63' => 'ipp042','ota64' => 'ipp043','ota65' => 'ipp044','ota66' => 'ipp046','ota67' => 'ipp047', + 'ota71' => 'ipp048','ota72' => 'ipp049','ota73' => 'ipp050','ota74' => 'ipp051','ota75' => 'ipp052','ota76' => 'ipp053'); + %{ $destination{NAME} } = ('ota01' => 'ipp010.0','ota02' => 'ipp011.0','ota03' => 'ipp006.0','ota04' => 'ipp054.0','ota05' => 'ipp007.0','ota06' => 'ipp055.0', + 'ota10' => 'ipp008.0','ota11' => 'ipp056.0','ota12' => 'ipp009.0','ota13' => 'ipp057.0','ota14' => 'ipp058.0','ota15' => 'ipp059.0','ota16' => 'ipp012.0','ota17' => 'ipp060.0', + 'ota20' => 'ipp013.0','ota21' => 'ipp014.0','ota22' => 'ipp061.0','ota23' => 'ipp015.0','ota24' => 'ipp016.0','ota25' => 'ipp062.0','ota26' => 'ipp064.0','ota27' => 'ipp065.0', + 'ota30' => 'ipp066.0','ota31' => 'ipp020.0','ota32' => 'ipp063.0','ota33' => 'ipp021.0','ota34' => 'ipp023.0','ota35' => 'ipp013.0','ota36' => 'ipp024.0','ota37' => 'ipp019.0', + 'ota40' => 'ipp025.0','ota41' => 'ipp018.0','ota42' => 'ipp017.0','ota43' => 'ipp015.0','ota44' => 'ipp027.0','ota45' => 'ipp028.0','ota46' => 'ipp029.0','ota47' => 'ipp030.0', + 'ota50' => 'ipp031.0','ota51' => 'ipp032.0','ota52' => 'ipp033.0','ota53' => 'ipp034.0','ota54' => 'ipp035.0','ota55' => 'ipp036.0','ota56' => 'ipp037.0','ota57' => 'ipp038.0', + 'ota60' => 'ipp039.0','ota61' => 'ipp040.0','ota62' => 'ipp041.0','ota63' => 'ipp042.0','ota64' => 'ipp043.0','ota65' => 'ipp044.0','ota66' => 'ipp046.0','ota67' => 'ipp047.0', + 'ota71' => 'ipp048.0','ota72' => 'ipp049.0','ota73' => 'ipp050.0','ota74' => 'ipp051.0','ota75' => 'ipp052.0','ota76' => 'ipp053.0'); + foreach my $k (keys %{ $destination{NAME} }) { + $destination{VOL}{$k} = $vol_id_list{$destination{NAME}{$k}}; + } + my %blocked_volume = ('ipp027' => 1, + 'ipp037' => 1, + 'ipp043' => 1, + 'ipp044' => 1, + 'ipp053' => 1, + 'ipp023' => 1,'ipp024' => 1,'ipp025' => 1,'ipp026' => 1,'ipp028' => 1,'ipp029' => 1,'ipp030' => 1,'ipp031' => 1,'ipp032' => 1,'ipp033' => 1, + 'ipp034' => 1,'ipp035' => 1,'ipp036' => 1,'ipp038' => 1,'ipp039' => 1,'ipp040' => 1,'ipp041' => 1,'ipp042' => 1,'ipp045' => 1,'ipp046' => 1, + 'ipp047' => 1,'ipp048' => 1,'ipp049' => 1,'ipp050' => 1,'ipp051' => 1,'ipp052' => 1, + # These hosts are blocked due to concern about their RAID arrays. +# 'ipp054' => 1,'ipp055' => 1,'ipp056' => 1,'ipp057' => 1,'ipp058' => 1,'ipp059' => 1, +# 'ipp060' => 1,'ipp061' => 1,'ipp062' => 1,'ipp063' => 1,'ipp064' => 1,'ipp065' => 1, 'ipp066' => 1 + 'ipp064' => 1, 'ipp066' => 1 + ); + if (not defined $so_id_start) { $so_id_start = 0; } + if (not defined $so_id_range) { $so_id_range = 100000; } + my $so_id_end = $so_id_start + $so_id_range; + + # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10 + { + my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object"); + $query->execute; + my $answer = $query->fetchrow_arrayref; + my $max_so_id = $$answer[0]; + $query->finish; + + if ($so_id_start > $max_so_id) { + print STDERR "at end of so_id range, reset please\n"; + exit 10; + } + } + + # Determine a list of first instances +# my $query = $dbh->prepare( +# " +# SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host FROM ( +# select * FROM ( +# select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id,value AS user_copies FROM storage_object +# JOIN storage_object_xattr USING(so_id) +# RIGHT JOIN instance USING(so_id) +# WHERE name = 'user.copies' +# AND value >= 2 +# AND ext_id LIKE '%ota%fits' +# AND so_id >= $so_id_start +# AND so_id < $so_id_end +# GROUP BY so_id +# ) AS V +# WHERE MAXins_id != MINins_id +# ) AS S +# JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) +# JOIN volume USING(vol_id) WHERE volume.available = 1 +# LIMIT $limit +# OFFSET $offset +# "); + my $query = $dbh->prepare( + " +SELECT Z.so_id,ext_id,Z.ins_id,Z.uri,substr(Z.uri,-10,5) AS class_id,Z.vol_id,vol_name,vol_host, + instance.vol_id AS second_vol_id,volume.name AS second_name,volume.host AS second_host FROM ( + SELECT S.so_id,ext_id,ins_id,uri,substr(uri,-10,5) AS class_id,vol_id,name AS vol_name,host AS vol_host,MAXins_id FROM ( + select * FROM ( + select MAX(ins_id) AS MAXins_id, MIN(ins_id) AS MINins_id, so_id,ext_id FROM storage_object + JOIN storage_object_xattr USING(so_id) + RIGHT JOIN instance USING(so_id) + WHERE name = 'user.copies' + AND value >= 2 + AND ext_id LIKE '%ota%fits' + AND so_id >= $so_id_start + AND so_id < $so_id_end + GROUP BY so_id + ) AS V + WHERE MAXins_id != MINins_id + ) AS S + JOIN instance ON (S.MINins_id = instance.ins_id AND S.so_id = instance.so_id) + JOIN volume USING(vol_id) WHERE volume.available = 1 + ) AS Z +JOIN instance ON (MAXins_id = instance.ins_id AND Z.so_id = instance.so_id) +JOIN volume ON(instance.vol_id = volume.vol_id) WHERE volume.available = 1 +LIMIT $limit +OFFSET $offset +"); + + $query->execute(); + + my @rows; + while (my $row = $query->fetchrow_hashref) { + push @rows, $row; + } + $query->finish; + + print STDERR "No rows found\n" unless scalar @rows; + exit unless scalar @rows; + + # compare number of responses to limit below + my $Npending = @rows; + + print "targetPending MULTI\n\n"; + + foreach my $obj (@rows) { + if (defined $verbose) { + require Data::Dumper; + print Data::Dumper::Dumper($obj); + } + + my $so_id = $obj->{so_id}; + my $key = $obj->{ext_id}; + + my $ins_id = $obj->{ins_id}; + my $uri = $obj->{uri}; + my $class_id = $obj->{class_id}; + + my $source_vol_id = $obj->{vol_id}; + my $source_name = $obj->{vol_name}; + my $source_host = $obj->{vol_host}; + + my $second_vol_id = $obj->{second_vol_id}; + my $second_vol_name = $obj->{second_name}; + my $second_vol_host = $obj->{second_host}; + + my $destination_vol_id = $destination{VOL}{$class_id}; + my $destination_name = $destination{NAME}{$class_id}; + my $destination_host = $destination{HOST}{$class_id}; + my $destination_uri = $uri; + unless ((defined($destination_uri))&& + (defined($source_name))&& + (defined($destination_name))) { + next; + } + + $destination_uri =~ s/$source_name/$destination_name/; + + my %md = ( + so_id => $so_id, + key => $key, + ins_id => $ins_id, + uri => $uri, + class_id => $class_id, + + source_vol_id => $source_vol_id, + source_name => $source_name, + source_host => $source_host, + + destination_vol_id => $destination_vol_id, + destination_name => $destination_name, + destination_host => $destination_host, + destination_uri => $destination_uri, + second_host => $second_vol_host, + second_vol_id => $second_vol_id +# blocked_host_bool => defined($blocked_volume{$destination_host}) + ); + + if ((defined($blocked_volume{$destination_host}))|| + ($source_vol_id == $destination_vol_id)|| + ($second_vol_id == $destination_vol_id)|| + ($source_name eq $destination_name)|| + ($second_vol_name eq $destination_name)) { +# print_metadata("targetFailed", \%md); + next; + } + + if ($source_vol_id != $destination_vol_id) { + print_metadata("targetPending", \%md); + print "\n"; + } + } + + # use a different exit status if we hit the limit (likely more files pending) + print STDERR "Which exit: $Npending $limit\n"; + # This is a better way to do this. This should be even better-er. + if ($Npending < $limit) { + exit 0; + } + else { + exit 1; + } +} + sub removal { Index: branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-host =================================================================== --- branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-host (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-host (revision 33415) @@ -18,5 +18,5 @@ my ($host, $state, $note, $db, $dbhost, $dbpass, $dbuser, - $allocate, $available, $debug); + $allocate, $available, $debug,$volume); # $db = $ENV{'NEB_DB'} unless $db; @@ -56,5 +56,5 @@ 'dbuser=s' => \$dbuser, 'dbpass=s' => \$dbpass, - + 'volume' => \$volume, ) || pod2usage( 2 ); @@ -98,5 +98,10 @@ my %constraint; - $constraint{'v.host'} = $host; + if (defined($volume)) { + $constraint{'v.name'} = $host; + } + else { + $constraint{'v.host'} = $host; + } my %set; $set{'v.allocate'} = $allocate if defined $allocate; Index: branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-insedit =================================================================== --- branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-insedit (revision 33415) +++ branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-insedit (revision 33415) @@ -0,0 +1,292 @@ +#!/usr/bin/env perl + +# Copyright (C) 2005-2008 Joshua Hoblitt +# +# $Id: neb-insedit,v 1.2 2008-10-01 21:05:57 jhoblitt Exp $ + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.02'; + +use DBI; +use Nebulous::Server::SQL; +use Nebulous::Client; +use URI::file; +use URI; + +use Digest::MD5; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +my ($db, $dbhost, $dbuser, $dbpass, $key, $ins_id, $new_uri, $new_volume,$nebhost); + +$db = $ENV{'NEB_DB'} unless $db; +$dbhost = $ENV{'NEB_DBHOST'} || 'localhost'; +$dbuser = $ENV{'NEB_USER'} unless $dbuser; +$dbpass = $ENV{'NEB_PASS'} unless $dbpass; +$nebhost= $ENV{'NEB_SERVER'} unless $nebhost; + +GetOptions( + 'db|d=s' => \$db, + 'host=s' => \$dbhost, + 'pass|p=s' => \$dbpass, + 'user|u=s' => \$dbuser, + 'neb_host=s' => \$nebhost, + 'key=s' => \$key, + 'ins_id=s' => \$ins_id, + 'uri=s' => \$new_uri, + 'volume=s' => \$new_volume, +) || pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --db --user --pass --key --ins_id --uri --volume", -exitval => 2 ) + unless $db && $dbuser && $dbpass && $ins_id && $new_uri && $key && $new_volume; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db:host=$dbhost", + $dbuser, + $dbpass, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 1, + }, +); + +# Connect and retrieve instance information via stat: +my $neb = Nebulous::Client->new( + proxy => "$nebhost", + ); + +die "can't connected to Nebulous Server: $nebhost" + unless defined $neb; + + +# Pre-emptively attempt repair of file. This will ~double our execution time, but should help the efficiency. +system("neb-repair $key"); +if ($? >> 8 != 0) { + die "Failure to repair $key.\n"; +} + +# Do checks now. + +my $stat = $neb->stat($key); +die "nebulous key: $key not found" unless $stat; +my $instances = $neb->find_instances($key, 'any'); +die "no instances found" unless $instances; + +my $user_copies; +# eval { +# $user_copies = $neb->getxattr($key, "user.copies"); +# }; +$user_copies = 1; +unless(defined($user_copies)) { + $user_copies = 1; +} +my @validation; +my %md5sum_uniq; +my $existing_copies = 0; +my @existance; +my @md5sums; +my @files = map {URI->new($_)->file if $_} @$instances; + +for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + $md5sum_uniq{$md5sums[$i]} = 1; + close($fh); + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + } + $validation[$i] = sprintf("% 3d %32s %s", + $existance[$i], + $md5sums[$i], + $files[$i]); +} +# Confirm everything is in a valid state +my $N_md5sums = scalar(keys %md5sum_uniq); +if ($N_md5sums != 1) { + die "Cannot modify this instance! storage_object $key in inconsistent state!"; +} +if ($#files < 1) { + die "Insufficient instances! $key!"; +} +my $test_md5sum = $md5sums[0]; + +# Copy instance to new uri +my $destination_file = URI->new($new_uri)->file; +my $source_file = $files[0]; +system("cp $source_file $destination_file"); +system("glockfile $destination_file xcld 0"); + +my $fh; +open($fh,$destination_file); +my $new_md5sum = Digest::MD5->new->addfile($fh)->hexdigest; +close($fh); + +if ($new_md5sum ne $test_md5sum) { + die "md5sum mismatch with new copy. $destination_file ($new_md5sum) != $source_file ($test_md5sum) Exiting!"; +} + +# Update database table +my $sql = Nebulous::Server::SQL->new(); + +my $query = $dbh->prepare( $sql->get_volume_by_name ); +$query->execute( $new_volume ); +my ($vol_id, $vol_name, $vol_host, $vol_path) = $query->fetchrow_array; +$query->finish; + +print "$new_volume $vol_id $vol_name $vol_host $vol_path $new_uri $ins_id\n"; +unless (defined($vol_id)) { + die "Invalid destination volume."; +} + +$query = $dbh->prepare( $sql->update_instance_uri ); +$query->execute( $vol_id, $new_uri, $ins_id ); +$query->finish; + +unlink($source_file); +print " OK\n"; + +__END__ + +=pod + +=head1 NAME + +neb-insedit - edit the instance information + +=head1 SYNOPSIS + + neb-insedit --key --ins_id --uri --volume + [--db ] [--user ] [--pass ] [--host + +Nebulous key for the storage object associated with the instance to move. + +=item * --ins_id + +Instance identifier that will be moved + +=item * --uri + +Final location desired for this instance. Should be of the form "file://data/ipp0XX.0/nebulous/aa/aa/ins_id.nebulous:directory:tree:file". + +=item * --volume + +Volume name that this instance will be moved to. + +=item * --db|-d + +Name of database (C) to create tables in. + +Optional if the appropriate environment variable is set. + +=item * --user|-u + +Username to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --pass|-p + +Password to authenticate with. + +Optional if the appropriate environment variable is set. + +=item * --host + +Name of host on which the database resides. + +=item * --neb_host + +Location of the client side server to connect to. + +Optional. Defaults to C. + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --db|-d + +=item * C + +Equivalent to --host + +=item * C + +Equivalent to --user|-u + +=item * C + +Equivalent to --pass|-p + +=item * C + +Equivalent to --neb_host + +=back + +=head1 CREDITS + +Josh Hoblitt and Chris Waters + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Joshua Hoblitt + +=head1 COPYRIGHT + +Copyright (C) 2005-2008 Joshua Hoblitt. All rights reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L + +=cut Index: branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-voladm =================================================================== --- branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-voladm (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous-Server/bin/neb-voladm (revision 33415) @@ -68,5 +68,5 @@ if (defined $allocate and $allocate !~ m/^[01]$/) or (defined $available and $available !~ m/^[01]$/) - or (defined $xattr and $xattr !~ m/^[01]$/); + or (defined $xattr and $xattr !~ m/^[012]$/); } Index: branches/meh_branches/ppstack_test/Nebulous/Build.PL =================================================================== --- branches/meh_branches/ppstack_test/Nebulous/Build.PL (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous/Build.PL (revision 33415) @@ -116,4 +116,5 @@ bin/neb-mv bin/neb-replicate + bin/neb-repair bin/neb-rm bin/neb-shift Index: branches/meh_branches/ppstack_test/Nebulous/MANIFEST =================================================================== --- branches/meh_branches/ppstack_test/Nebulous/MANIFEST (revision 32161) +++ branches/meh_branches/ppstack_test/Nebulous/MANIFEST (revision 33415) @@ -17,4 +17,5 @@ bin/neb-mv bin/neb-replicate +bin/neb-repair bin/neb-rm bin/neb-shift Index: branches/meh_branches/ppstack_test/Nebulous/bin/neb-repair =================================================================== --- branches/meh_branches/ppstack_test/Nebulous/bin/neb-repair (revision 33415) +++ branches/meh_branches/ppstack_test/Nebulous/bin/neb-repair (revision 33415) @@ -0,0 +1,241 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007-2008 Chris Waters +# + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +use Digest::MD5; +use URI; + +my ($validate,$server,$nouris); + +$server = $ENV{'NEB_SERVER'} unless $server; + +$validate = 1; +$nouris = 1; + +GetOptions( + 'server|s=s' => \$server, +) || pod2usage( 2 ); + +my $key = shift; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) + unless $server; +pod2usage( -msg => "missing key", exitval => 2 ) + unless defined $key; + +my $neb = Nebulous::Client->new( + proxy => "$server", +); + +die "can't connected to Nebulous Server: $server" + unless defined $neb; + +my $stat = $neb->stat($key); +die "nebulous key: $key not found" unless $stat; +my $instances = $neb->find_instances($key, 'any'); +die "no instances found" unless $instances; + +my $user_copies; +eval { + $user_copies = $neb->getxattr($key, "user.copies"); +}; +unless(defined($user_copies)) { + $user_copies = 2; +} + +my @validation; +my %md5sum_uniq; +my %md5sum_valid; +my $last_f = ''; +my $existing_copies = 0; +my @existance; +my @md5sums; +my @files = map {URI->new($_)->file if $_} @$instances; + +for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + close($fh); + + $md5sum_uniq{$md5sums[$i]} = 1; + + if ($md5sums[$i] eq 'd41d8cd98f00b204e9800998ecf8427e') { + $existance[$i] = 0; + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + next; + } + + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + $last_f = $files[$i]; + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + } +} + +my $N_mds = scalar(keys(%md5sum_uniq)); +my $N_files = scalar(@files); +if ($N_files < $user_copies) { + print "Insufficient copies ($N_files) of $key, replicating!\n"; + system("neb-replicate $key"); +} +if ($N_mds == 1) { + print "Nebulous key $key is consistent.\n"; +} +else { + my $valid_sum = ''; + my $valid_file = ''; + my $m; + foreach $m (keys %md5sum_valid) { + if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) { + if ($valid_sum ne '') { + die "Too many acceptable files for key $key\n"; + } + $valid_sum = $m; + $valid_file = ${ $md5sum_valid{$m} }[0]; + } + } + if ($valid_sum eq '') { + die "No acceptable file for key $key\n"; + } + else { + print "Repairing instances\n"; + foreach $m (keys %md5sum_valid) { + if ($m ne $valid_sum) { + foreach my $f (@{ $md5sum_valid{$m} }) { + print "\tcp $valid_file $f\n"; + system("cp $valid_file $f"); + if ($? >> 8 != 0) { + if ($existance[0] == 1) { + # We can fix this with culls and replications. + print "Attempting to fix this.\n"; + print("neb-replicate $key\n"); + system("neb-replicate $key"); + if ($? >> 8 == 0) { + my $bad_volume = (split /\//, $f)[2]; + print("neb-cull --volume $bad_volume $key\n"); + system("neb-cull --volume $bad_volume $key"); + } + else { + die "Failed to copy file $valid_file to $f, and subsequent replication also failed!\n"; + } + } + else { + die "Failed to copy file $valid_file to $f!\n"; + } + } + } + } + } + } +} + +# if ($N_files > $user_copies) { +# my @split = split /\//, $last_f; +# my $volume = $split[2]; +# print "Culling extra copy: neb-cull --volume $volume $key\n"; +# system("neb-cull --volume $volume $key"); +# } + + + +__END__ + +=pod + +=head1 NAME + +neb-repair - tool to check and repair instances + +=head1 SYNOPSIS + + neb-repair [--server ] + +=head1 DESCRIPTION + +This program repairs instances and corrects instance counts for a nebulous key. + +=head1 OPTIONS + +=over 4 + +=item * --server|-s + +URL of the Nebulous server to connect to. + +Optional if the appropriate environment variable is set. + +=back + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --server|-s + +=back + +=head1 CREDITS + + + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Chris Waters + +=head1 COPYRIGHT + +Copyright (C) 2007-2008 Chris Waters + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, +L, C + +=cut Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/doc/wise.catalog.txt =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/doc/wise.catalog.txt (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/doc/wise.catalog.txt (revision 33415) @@ -0,0 +1,276 @@ +designation %20s ---- no Sexagesimal, equatorial position-based source name in the form: hhmmss.ss+ddmmss.s. The full naming convention for WISE Preliminary Release Catalog sources has the form "WISEP Jhhmmss.ss+ddmmss.s," where "WISEP" indicates the source is from the Preliminary Release Source Catalog, and "J" indicates the position is J2000. The "WISEP" acronym is not listed explicitly in the designation column. +ra %10.6f deg no J2000 right ascension with respect to the 2MASS PSC reference frame +dec %10.6f deg no J2000 declination with respect to the 2MASS PSC reference frame. +sigra %4.2f arcsec no One-sigma uncertainty in right ascension coordinate. +sigdec %4.2f arcsec no One-sigma uncertainty in declination coordinate. +sigradec %3d deg no The co-sigma of the equatorial position uncertainties, sig_ra, sig_dec (σα, σΎ). The co-sigma is related to the covariance between the position errors VαΎ by the formula VαΎ = σαΎ|σαΎ|. This maintains the sign information for the correlation, since σαΎ may be negative. It is more natural to carry the co-sigma along with the other uncertainties instead of the covariance because the former is in the same units as the other uncertainties. +glon %7.3f deg no Galactic longitude. CAUTION: This coordinate should not be used as an astrometric reference. +glat %7.3f deg no Galactic latitude. CAUTION: This coordinate should not be used as an astrometric reference. +elon %7.3f deg no Ecliptic longitude. CAUTION: This coordinate should not be used as an astrometric reference. +elat %7.3f deg no Ecliptic latitude. CAUTION: This coordinate should not be used as an astrometric reference. +wx %7.3f pix no The x-pixel coordinate of this source on the Atlas Image. +wy %7.3f pix no The y-pixel coordinate of this source on the Atlas Image. +cntr %19d ---- no Unique identification number for the Catalog source. This number is formed entirely from the source_id, which is in turn formed from the coadd_id and source number src. On average, sources with cntr values close to each other are also close to each other on the sky, except at Tile boundaries. Cntr ordering thus provides a useful mapping of the WISE Catalog on storage media. +source_id %28s ---- no Unique source ID, formed from a combination of the Atlas Tile ID, coadd_id, and sequential extracted source number, src, within the Tile. The source_id string has the form RRRRsDDD_[trev]-IIIIII where: +coadd_id %20s ---- no Atlas Tile identifier from which source was extracted. The identifier has the general form: RRRRsDDD_[trev], where +src %6d ---- no Sequential number of this source extraction in the Atlas Tile from which this source was extracted, in approximate descending order of W1 source brightness. +w1mpro %7.3f mag yes W1 magnitude measured with profile-fitting photometry, or the magnitude of the 95% confidence brightness upper limit if the W1 flux measurement has SNR<2. This column is null if the source is nominally detected in W1, but no useful brightness estimate could be made. +w1sigmpro %5.3f mag yes W1 profile-fit photometric measurement uncertainty in mag units. This column is null if the W1 profile-fit magnitude is a 95% confidence upper limit or if the source is not measurable. +w1snr %7.1f ---- yes W1 profile-fit measurement signal-to-noise ratio. This value is the ratio of the flux (w1flux) to flux uncertainty (w1sigflux)in the W1 profile-fit photometry measurement. This column is null if w1flux is negative, or if w1flux or w1sigflux are null. +w1rchi2 %9.3e ---- yes Reduced χ2 of the W1 profile-fit photometry measurement. This column is null if the W1 magnitude is a 95% confidence upper limit (i.e. the source is not detected). +w2mpro %7.3f mag yes W2 magnitude measured with profile-fitting photometry, or the magnitude of the 95% confidence brightness upper limit if the W2 flux measurement has SNR<2. This column is null if the source is nominally detected in W1, but no useful brightness estimate could be made. +w2sigmpro %5.3f mag yes W2 profile-fit photometric measurement uncertainty in mag units. This column is null if the W2 profile-fit magnitude is a 95% confidence upper limit or if the source is not measurable. +w2snr %7.1f ---- yes W2 profile-fit measurement signal-to-noise ratio. This value is the ratio of the flux (w2flux to flux uncertainty (w2sigflux)in the W2 profile-fit photometry measurement. This column is null if w2flux is negative, or if w2flux or w2sigflux are null. +w2rchi2 %9.3e ---- yes Reduced χ2 of the W2 profile-fit photometry measurement. This column is null if the W2 magnitude is a 95% confidence upper limit (i.e. the source is not detected). +w3mpro %7.3f mag yes W3 magnitude measured with profile-fitting photometry, or the magnitude of the 95% confidence brightness upper limit if the W3 flux measurement has SNR<2. This column is null if the source is nominally detected in W3, but no useful brightness estimate could be made. +w3sigmpro %5.3f mag yes W3 profile-fit photometric measurement uncertainty in mag units. This column is null if the W3 profile-fit magnitude is a 95% confidence upper limit or if the source is not measurable. +w3snr %7.1f ---- yes W3 profile-fit measurement signal-to-noise ratio. This value is the ratio of the flux (w3flux to flux uncertainty (w3sigflux)in the W3 profile-fit photometry measurement. This column is null if w3flux is negative, or if w3flux or w3sigflux are null. +w3rchi2 %9.3e ---- yes Reduced χ2 of the W3 profile-fit photometry measurement. This column is null if the W3 magnitude is a 95% confidence upper limit (i.e. the source is not detected). +w4mpro %7.3f mag yes W4 magnitude measured with profile-fitting photometry, or the magnitude of the 95% confidence brightness upper limit if the W4 flux measurement has SNR<2. This column is null if the source is nominally detected in W4, but no useful brightness estimate could be made. +w4sigmpro %5.3f mag yes W4 profile-fit photometric measurement uncertainty in mag units. This column is null if the W4 profile-fit magnitude is a 95% confidence upper limit or if the source is not measurable. +w4snr %7.1f ---- yes W4 profile-fit measurement signal-to-noise ratio. This value is the ratio of the flux (w4flux to flux uncertainty (w4sigflux)in the W4 photometry measurement. This column is null if w4flux is negative, or if w4flux or w4sigflux are null. +w4rchi2 %9.3e ---- yes Reduced χ2 of the W4 profile-fit photometry measurement. This column is null if the W4 magnitude is a 95% confidence upper limit (i.e. the source is not detected). +rchi2 %9.3f ---- yes Combine reduced χ2 of the profile-fit photometry measurement in all bands. +nb %1d ---- yes Number of PSF components used simultaneously in the profile-fitting for this source. This number includes the source itself, so the minimum value of nb is "1". Nb is greater than "1" when the source is fit concurrently with other nearby detections (passive deblending), or when a single source is split into two components during the fitting process (active deblending). +na %1d ---- yes Active deblending flag. Indicates if a single detection was split into multiple sources in the process of profile-fitting. +w1sat %5.3f ---- yes Saturated pixel fraction, W1. The fraction of all pixels within the profile-fitting area in the stack of single-exposure images used to characterize this source that are flagged as saturated. A value larger than 0.0 indicates one or more pixels of saturation. Saturation begins to occur for point sources brighter than [W1]~8 mag. Saturation may occur in fainter sources because of a charged particle strike. +w2sat %5.3f ---- yes Saturated pixel fraction, W2. The fraction of all pixels within the profile-fitting area in the stack of single-exposure images used to characterize this source that are flagged as saturated. A value larger than 0.0 indicates one or more pixels of saturation. Saturation begins to occur for point sources brighter than [W2]~7 mag. Saturation may occur in fainter sources because of a charged particle strike. +w3sat %5.3f ---- yes Saturated pixel fraction, W3. The fraction of all pixels within the profile-fitting area source in the stack of single-exposure images used to characterize this source that are flagged as saturated. A value larger than 0.0 indicates one or more pixels of saturation. Saturation begins to occur for point sources brighter than [W3]~4 mag. Saturation may occur in fainter sources because of a charged particle strike. +w4sat %5.3f ---- yes Saturated pixel fraction, W4. The fraction of all pixels within the profile-fitting area in the stack of single-exposure images used to characterize this source that are flagged as saturated. A value larger than 0.0 indicates one or more pixels of saturation. Saturation begins to occur for point sources brighter than [W4]~0 mag. Saturation may occur in fainter sources because of a charged particle strike. +satnum %4s ---- no Minimum sample at which saturation occurs in each band. Four character string, one character per band, that indicates the minimum SUTR sample in which any pixel in the profile-fitting area in all of the single-exposure images used to characterize this source was flagged as having reached the saturation level in the on-board WISE payload processing. If no pixels in a given band are flagged as saturated, the value for that band is "0". +cc_flags %4s ---- no Contamination and confusion flag. Four character strong, one character per band [W1/W2/W3/W4], that indicates that the photometry and/or position measurements of a source may be contaminated or biased due to proximity to an image artifact. +ext_flg %1d ---- no Extended source flag. This flag indicates whether or not the morphology of a source is consistent with the WISE point spread function in any band. +var_flg %4s ---- yes Variability flag. The variability flag is a four-character string, one character per band, in which the value for each band is related to the probability that the source flux measured on the individual WISE exposures was not constant with time. +ph_qual %4s ---- no + +det_bit %3d ---- no Bit-encoded integer indicating bands in which a source has a w?snr>2 detection. For example, a source detected in W1 only has det_bit=1 (binary 0001). A source detected in W4 only has det_bit=8 (binary 1000). A source detected in all four bands has det_bit=15 (binary 1111). +w1nm %6d ---- yes Integer frame detection count. This column gives the number of individual 7.7s exposures on which this source was detected with SNR>3 in the W1 profile-fit measurement. This number can be zero for sources that are well-detected on the coadded Atlas Image, but too faint for detection on the single exposures. +w1m %6d ---- yes Integer frame coverage. This column gives the number of individual 7.7s W1 exposures on which a profile-fit measurement of this source was possible. +w2nm %6d ---- yes Integer frame detection count. This column gives the number of individual 7.7s exposures on which this source was detected with SNR>3 in the W2 profile-fit measurement. This number can be zero for sources that are well-detected on the coadded Atlas Image, but too faint for detection on the single exposures. +w2m %6d ---- yes Integer frame coverage. This column gives the number of individual 8.8s W2 exposures on which a profile-fit measurement of this source was possible. + +w3nm %6d ---- yes Integer frame detection count. This column gives the number of individual 8.8s exposures on which this source was detected with SNR>3 in the W3 profile-fit measurement. This number can be zero for sources that are well-detected on the coadded Atlas Image, but too faint for detection on the single exposures. +w3m %6d ---- yes Integer frame coverage. This column gives the number of individual 7.7s W3 exposures on which a profile-fit measurement of this source was possible. +w4nm %6d ---- yes Integer frame detection count. This column gives the number of individual 8.8s exposures on which this source was detected with SNR>3 in the W4 profile-fit measurement. This number can be zero for sources that are well-detected on the coadded Atlas Image, but too faint for detection on the single exposures. +w4m %6d ---- yes Integer frame coverage. This column gives the number of individual 8.8s W4 exposures on which a profile-fit measurement of this source was possible. +w1cov %7.3f ---- yes Mean pixel coverage in W1. This column gives the mean pixel value from the W1 Atlas Tile Coverage Map within the "standard" aperture, a circular area with a radius of 8.25" centered on the position of this source. + +w2cov %7.3f ---- yes Mean pixel coverage in W2. This column gives the mean pixel value from the W2 Atlas Tile Coverage Map within the "standard" aperture, a circular area with a radius of 8.25" centered on the position of this source. +w3cov %7.3f ---- yes Mean pixel coverage in W3. This column gives the mean pixel value from the W3 Atlas Tile Coverage Map within the "standard" aperture, a circular area with a radius of 8.25" centered on the position of this source. +w4cov %7.3f ---- yes Mean pixel coverage in W4. This column gives the mean pixel value from the W4 Atlas Tile Coverage Map within the "standard" aperture, a circular area with a radius of 17.0" centered on the position of this source. +w1frl %5.3f ---- yes Fraction of pixels affected by latents. This column gives the fraction of all W1 pixels in the stack of individual W1 exposures used to characterize this source that may be affected by the presence of a latent image produced by a bright source. This number is computed by counting the number of pixels in the single exposure Bit Mask images with value "25" that are present within the profile-fitting area, a circular region with radius of 7.25", centered on the position of this source, and dividing by the total number of pixels in the same area that are available for measurement. +w2frl %5.3f ---- yes Fraction of pixels affected by latents. This column gives the fraction of all W2 pixels in the stack of individual W2 exposures used to characterize this source that may be affected by the presence of a latent image produced by a bright source. This number is computed by counting the number of pixels in the single exposure Bit Mask images with value "25" that are present within the profile-fitting area, a circular region with radius of 7.25", centered on the position of this source, and dividing by the total number of pixels in the same area that are available for measurement. + +w3frl %5.3f ---- yes Fraction of pixels affected by latents. This column gives the fraction of all W3 pixels in the stack of individual W3 exposures used to characterize this source that may be affected by the presence of a latent image produced by a bright source. This number is computed by counting the number of pixels in the single exposure Bit Mask images with value "25" that are present within the profile-fitting area, a circular region with radius of 7.25", centered on the position of this source, and dividing by the total number of pixels in the same area that are available for measurement. +w4frl %5.3f ---- yes Fraction of pixels affected by latents. This column gives the fraction of all W4 pixels in the stack of individual W4 exposures used to characterize this source that may be affected by the presence of a latent image produced by a bright source. This number is computed by counting the number of pixels in the single exposure Bit Mask images with value '25' that are present within the profile-fitting area, a circular region with radius of 15.0", centered on the position of this source, and dividing by the total number of pixels in the same area that are available for measurement. +w1cc_map %3d ---- no Contamination and confusion map for this source in W1. This column contains the integer equivalent of the 9-bit binary number that specifies if the W1 measurement is believed to be contaminated by or a spurious detection of an image artifact. The elements of the binary array are: +w1cc_map_str %12s ---- yes Contamination and confusion string. This column is a character string that denotes all artifacts that may contaminate the W1 measurement of this source, in the priority order D,P,H,O. Lower-case letters denote cases where the source detection in this band is believed to be real but the measurement may be contaminated by the artifact. upper-case letters denote cases where the source detection in this band may be a spurious detection of an artifact. +w2cc_map %3d ---- no Contamination and confusion map for this source in W2. This column contains the integer equivalent of the 9-bit binary number that specifies if the W2 measurement is believed to be contaminated by or a spurious detection of an image artifact. The elements of the binary array are: + +w2cc_map_str %12s ---- yes +w3cc_map %3d ---- no Contamination and confusion map for this source in W3. This column contains the integer equivalent of the 9-bit binary number that specifies if the W3 measurement is believed to be contaminated by or a spurious detection of an image artifact. The elements of the binary array are: +w3cc_map_str %12s ---- yes Contamination and confusion string. This column is a character string that denotes all artifacts that may contaminate the W3 measurement of this source, in the priority order D,P,H,O. Lower-case letters denote cases where the source detection in this band is believed to be real but the measurement may be contaminated by the artifact. Upper-case letters denote cases where the source detection in this band may be a spurious detection of an artifact. +w4cc_map %3d ---- no Contamination and confusion map for this source in W4. This column contains the integer equivalent of the 9-bit binary number that specifies if the W4 measurement is believed to be contaminated by or a spurious detection of an image artifact. The elements of the binary array are: +w4cc_map_str %12s ---- yes Contamination and confusion string. This column is a character string that denotes all artifacts that may contaminate the W4 measurement of this source, in the priority order D,P,H,O. Lower-case letters denote cases where the source detection in this band is believed to be real but the measurement may be contaminated by the artifact. Upper-case letters denote cases where the source detection in this band may be a spurious detection of an artifact. +- skip 25 felds + +w1flux %9.3f dn yes The "raw" W1 source flux measured in profile-fit photometry in units of digital numbers. This value may be negative. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w1sigflux %9.3f dn yes Uncertainty in the "raw" W1 source flux measurement in profile-fit photometry in units of digital numbers. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w1sky %9.3f dn yes The trimmed average of the W1 sky background value in digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". Both profile-fit and aperture photometry source brightness measurements are made relative to this sky background value. +w1sigsk %7.3f dn yes The uncertainty in the W1 sky background value in units of digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". +w1conf %9.3f dn yes Estimated confusion noise in the W1 sky background annulus, in digital numbers. This number is the difference between the measured noise in the sky background w1sigsk and the noise measured in the same region on the Atlas Uncertainty Maps. +w2flux %9.3f dn yes The "raw" W2 source flux measured in profile-fit photometry in units of digital numbers. This value may be negative. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w2sigflux %9.3f dn yes Uncertainty in the "raw" W2 source flux measurement in profile-fit photometry in units of digital numbers. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w2sky %9.3f dn yes The trimmed average of the W2 sky background value in digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". Both profile-fit and aperture photometry source brightness measurements are made relative to this sky background value. +w2sigsk %7.3f dn yes The uncertainty in the W2 sky background value in units of digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". +w2conf %9.3f dn yes Estimated confusion noise in the W2 sky background annulus, in digital numbers. This number is the difference between the measured noise in the sky background w2sigsk and the noise measured in the same region on the Atlas Uncertainty Maps. +w3flux %9.3f dn yes The "raw" W3 source flux measured in profile-fit photometry in units of digital numbers. This value may be negative. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w3sigflux %9.3f dn yes Uncertainty in the "raw" W3 source flux measurement in profile-fit photometry in units of digital numbers. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w3sky %9.3f dn yes The trimmed average of the W3 sky background value in digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". Both profile-fit and aperture photometry source brightness measurements are made relative to this sky background value. +w3sigsk %7.3f dn yes The uncertainty in the W3 sky background value in units of digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". +w3conf %9.3f dn yes Estimated confusion noise in the W3 sky background annulus, in digital numbers. This number is the difference between the measured noise in the sky background w3sigsk and the noise measured in the same region on the Atlas Uncertainty Maps. +w4flux %9.3f dn yes The "raw" W1 source flux measured in profile-fit photometry in units of digital numbers. This value may be negative. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w4sigflux %9.3f dn yes Uncertainty in the "raw" W1 source flux measurement in profile-fit photometry in units of digital numbers. This column is null if no useful profile-fit measurement of the source is possible because of masked or otherwise unusable pixels. +w4sky %9.3f dn yes The trimmed average of the W1 sky background value in digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". Both profile-fit and aperture photometry source brightness measurements are made relative to this sky background value. +w4sigsk %7.3f dn yes The uncertainty in the W1 sky background value in units of digital numbers measured in an annulus with an inner radius of 50" and outer radius of 70". +w4conf %9.3f dn yes Estimated confusion noise in the W1 sky background annulus, in digital numbers. This number is the difference between the measured noise in the sky background w4sigsk and the noise measured in the same region on the Atlas Uncertainty Maps. +- skip 20 fields + +w1mag %7.3f mag yes W1 "standard" aperture magnitude. This is the curve-of-growth corrected source brightness measured in an 8.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". The curve-of-growth correction is given in w1mcor. +w1sigm %5.3f mag yes Uncertainty in the W1 "standard" aperture magnitude. +w1flg %3d ---- yes W1 "standard" aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag value is an integer that is the combination of one or more of the following values that signify different conditions: +w1mcor %6.3f mag yes W1 aperture curve-of-growth correction, in magnitudes. This correction is subtracted from the nominal 8.25" aperture photometry brightness, w1mag_2, to give the "standard-aperture" magnitude. + +w2mag %7.3f mag yes W2 "standard" aperture magnitude. This is the curve-of-growth corrected source brightness measured in an 8.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". The curve-of-growth correction is given in w2mcor. +w2sigm %5.3f mag yes Uncertainty in the W2 "standard" aperture magnitude. +w2flg %3d ---- yes W2 "standard" aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag value is an integer that is the combination of one or more of the following values that signify different conditions: +w2mcor %6.3f mag yes W2 aperture curve-of-growth correction, in magnitudes. This correction is subtracted from the nominal 8.25" aperture photometry brightness, w2mag_2, to give the "standard-aperture" magnitude. + +w3mag %7.3f mag yes W3 "standard" aperture magnitude. This is the curve-of-growth corrected source brightness measured in an 8.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". The curve-of-growth correction is given in w3mcor. +w3sigm %5.3f mag yes Uncertainty in the W3 "standard" aperture magnitude. +w3flg %3d ---- yes W3 "standard" aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag value is an integer that is the combination of one or more of the following values that signify different conditions: +w3mcor %6.3f mag yes W3 aperture curve-of-growth correction, in magnitudes. This correction is subtracted from the nominal 8.25" aperture photometry brightness, w3mag_2, to give the "standard-aperture" magnitude. + +w4mag %7.3f mag yes W4 "standard" aperture magnitude. This is the curve-of-growth corrected source brightness measured in a 16.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". The curve-of-growth correction is given in w4mcor. +w4sigm %5.3f mag yes Uncertainty in the W4 "standard" aperture magnitude. +w4flg %3d ---- yes W4 "standard" aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag value is an integer that is the combination of one or more of the following values that signify different conditions: +w4mcor %6.3f mag yes W4 aperture curve-of-growth correction, in magnitudes. This correction is subtracted from the nominal 16.5" aperture photometry brightness, w4mag_2, to give the "standard-aperture" magnitude. +- 16 fields + +w1mag_1 %7.3f mag yes W1 5.5" radius aperture magnitude. This is the calibrated source brightness measured in a 5.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_1 %5.3f mag yes Uncertainty in the W1 5.5" radius aperture magnitude. +w1flg_1 %3d ---- yes W1 5.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_1 %7.3f mag yes W2 5.5" radius aperture magnitude. This is the calibrated source brightness measured in a 5.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_1 %5.3f mag yes Uncertainty in the W2 5.5" radius aperture magnitude. +w2flg_1 %3d ---- yes W2 5.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_1 %7.3f mag yes W3 5.5" radius aperture magnitude. This is the calibrated source brightness measured in a 5.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_1 %5.3f mag yes Uncertainty in the W3 5.5" radius aperture magnitude. +w3flg_1 %3d ---- yes W3 5.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_1 %7.3f mag yes W4 11.0" radius aperture magnitude. This is the calibrated source brightness measured in a 11.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_1 %5.3f mag yes Uncertainty in the W4 11.0" radius aperture magnitude. +w4flg_1 %3d ---- yes W4 11.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_2 %7.3f mag yes W1 8.25" radius aperture magnitude. This is the calibrated source brightness measured in a 8.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_2 %5.3f mag yes Uncertainty in the W1 8.25" radius aperture magnitude. +w1flg_2 %3d ---- yes W1 8.25" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_2 %7.3f mag yes W2 8.25" radius aperture magnitude. This is the calibrated source brightness measured in a 8.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_2 %5.3f mag yes Uncertainty in the W2 8.25" radius aperture magnitude. +w2flg_2 %3d ---- yes W2 8.25" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_2 %7.3f mag yes W3 8.25" radius aperture magnitude. This is the calibrated source brightness measured in a 8.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_2 %5.3f mag yes Uncertainty in the W3 8.25" radius aperture magnitude. +w3flg_2 %3d ---- yes W3 8.25" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_2 %7.3f mag yes W4 16.5" radius aperture magnitude. This is the calibrated source brightness measured in a 16.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_2 %5.3f mag yes Uncertainty in the W4 16.5" radius aperture magnitude. +w4flg_2 %3d ---- yes W4 16.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_3 %7.3f mag yes W1 11.0" radius aperture magnitude. This is the calibrated source brightness measured in a 11.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_3 %5.3f mag yes Uncertainty in the W1 11.0" radius aperture magnitude. +w1flg_3 %3d ---- yes W1 11.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_3 %7.3f mag yes W2 11.0" radius aperture magnitude. This is the calibrated source brightness measured in a 11.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_3 %5.3f mag yes Uncertainty in the W2 11.0" radius aperture magnitude. +w2flg_3 %3d ---- yes W2 11.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_3 %7.3f mag yes W3 11.0" radius aperture magnitude. This is the calibrated source brightness measured in a 11.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_3 %5.3f mag yes Uncertainty in the W3 11.0" radius aperture magnitude. +w3flg_3 %3d ---- yes W3 11.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_3 %7.3f mag yes W4 22.0" radius aperture magnitude. This is the calibrated source brightness measured in a 22.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_3 %5.3f mag yes Uncertainty in the W4 22.0" radius aperture magnitude. +w4flg_3 %3d ---- yes W4 22.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_4 %7.3f mag yes W1 13.75" radius aperture magnitude. This is the calibrated source brightness measured in a 13.75" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_4 %5.3f mag yes Uncertainty in the W1 13.75" radius aperture magnitude. +w1flg_4 %3d ---- yes W1 13.75" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_4 %7.3f mag yes W2 13.75" radius aperture magnitude. This is the calibrated source brightness measured in a 13.75" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_4 %5.3f mag yes Uncertainty in the W2 13.75" radius aperture magnitude. +w2flg_4 %3d ---- yes W2 13.75" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_4 %7.3f mag yes W3 13.75" radius aperture magnitude. This is the calibrated source brightness measured in a 13.75" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_4 %5.3f mag yes Uncertainty in the W3 13.75" radius aperture magnitude. +w3flg_4 %3d ---- yes W3 13.75" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_4 %7.3f mag yes W4 27.5" radius aperture magnitude. This is the calibrated source brightness measured in a 27.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_4 %5.3f mag yes Uncertainty in the W4 27.5" radius aperture magnitude. +w4flg_4 %3d ---- yes W4 27.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_5 %7.3f mag yes W1 16.5" radius aperture magnitude. This is the calibrated source brightness measured in a 16.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_5 %5.3f mag yes Uncertainty in the W1 16.5" radius aperture magnitude. +w1flg_5 %3d ---- yes W1 16.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_5 %7.3f mag yes W2 16.5" radius aperture magnitude. This is the calibrated source brightness measured in a 16.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_5 %5.3f mag yes Uncertainty in the W2 16.5" radius aperture magnitude. +w2flg_5 %3d ---- yes W2 16.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_5 %7.3f mag yes W3 16.5" radius aperture magnitude. This is the calibrated source brightness measured in a 16.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_5 %5.3f mag yes Uncertainty in the W3 16.5" radius aperture magnitude. +w3flg_5 %3d ---- yes W3 16.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_5 %7.3f mag yes W4 33.0" radius aperture magnitude. This is the calibrated source brightness measured in a 33.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_5 %5.3f mag yes Uncertainty in the W4 33.0" radius aperture magnitude. +w4flg_5 %3d ---- yes W4 33.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_6 %7.3f mag yes W1 19.25" radius aperture magnitude. This is the calibrated source brightness measured in a 19.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_6 %5.3f mag yes Uncertainty in the W1 19.25" radius aperture magnitude. +w1flg_6 %3d ---- yes W1 19.25" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_6 %7.3f mag yes W2 19.25" radius aperture magnitude. This is the calibrated source brightness measured in a 19.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_6 %5.3f mag yes Uncertainty in the W2 19.25" radius aperture magnitude. +w2flg_6 %3d ---- yes W2 19.25" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_6 %7.3f mag yes W3 19.25" radius aperture magnitude. This is the calibrated source brightness measured in a 19.25" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_6 %5.3f mag yes Uncertainty in the W3 19.25" radius aperture magnitude. +w3flg_6 %3d ---- yes W3 19.25" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_6 %7.3f mag yes W4 38.5" radius aperture magnitude. This is the calibrated source brightness measured in a 38.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_6 %5.3f mag yes Uncertainty in the W4 38.5" radius aperture magnitude. +w4flg_6 %3d ---- yes W4 38.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_7 %7.3f mag yes W1 22.0" radius aperture magnitude. This is the calibrated source brightness measured in a 22.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_7 %5.3f mag yes Uncertainty in the W1 22.0" radius aperture magnitude. +w1flg_7 %3d ---- yes W1 22.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_7 %7.3f mag yes W2 22.0" radius aperture magnitude. This is the calibrated source brightness measured in a 22.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_7 %5.3f mag yes Uncertainty in the W2 22.0" radius aperture magnitude. +w2flg_7 %3d ---- yes W2 22.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_7 %7.3f mag yes W3 22.0" radius aperture magnitude. This is the calibrated source brightness measured in a 22.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_7 %5.3f mag yes Uncertainty in the W3 22.0" radius aperture magnitude. +w3flg_7 %3d ---- yes W3 22.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_7 %7.3f mag yes W4 44.0" radius aperture magnitude. This is the calibrated source brightness measured in a 44.0" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_7 %5.3f mag yes Uncertainty in the W4 44.0" radius aperture magnitude. +w4flg_7 %3d ---- yes W4 44.0" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1mag_8 %7.3f mag yes W1 24.75" radius aperture magnitude. This is the calibrated source brightness measured in a 24.75" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w1sigm_8 %5.3f mag yes Uncertainty in the W1 24.75" radius aperture magnitude. +w1flg_8 %3d ---- yes W1 24.75" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w1mflg. +w2mag_8 %7.3f mag yes W2 24.75" radius aperture magnitude. This is the calibrated source brightness measured in a 24.75" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w2sigm_8 %5.3f mag yes Uncertainty in the W2 24.75" radius aperture magnitude. +w2flg_8 %3d ---- yes W2 24.75" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w2mflg. +w3mag_8 %7.3f mag yes W3 24.75" radius aperture magnitude. This is the calibrated source brightness measured in a 24.75" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w3sigm_8 %5.3f mag yes Uncertainty in the W3 24.75" radius aperture magnitude. +w3flg_8 %3d ---- yes W3 24.75" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w3mflg. +w4mag_8 %7.3f mag yes W4 49.5" radius aperture magnitude. This is the calibrated source brightness measured in a 49.5" radius circular aperture centered on the source position on the Atlas Image. If the source is not detected in the aperture measurement, this is the 95% confidence upper limit to the brightness. The background sky reference level is measured in an annular region with inner radius of 50" and outer radius of 70". No curve-of-growth correction has been applied. +w4sigm_8 %5.3f mag yes Uncertainty in the W4 49.5" radius aperture magnitude. +w4flg_8 %3d ---- yes W4 49.5" radius aperture magnitude quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the standard aperture photometry quality flag, w4mflg. +- 12 fields +w1magp %6.3f mag yes Inverse-variance-weighted mean W1 magnitude computed from profile-fit measurements on the w1m individual frames covering this source. This differs from w1mpro in that it is computed by combining the profile-fit measurements from individual frames, whereas w1mpro is computed by fitting all W1 frames simultaneously and incorporating a robust error model. +w1sigp1 %6.3f mag yes Standard deviation of the population of W1 fluxes measured on the w1m individual frames covering this source, in magnitudes. This provides a measure of the characteristic uncertainty of the measurement of this source on individual frames. +w1sigp2 %6.3f mag yes Standard deviation of the mean of the distribution of W1 fluxes (w1magp) computed from profile-fit measurements on the w1m individual frames covering this source, in magnitudes. This is equivalent to w1sigp1/sqrt(w1m). +w2magp %6.3f mag yes Inverse-variance-weighted mean W2 magnitude computed from profile-fit measurements on the w2m individual frames covering this source. This differs from w2mpro in that it is computed by combining the profile-fit measurements from individual frames, whereas w2mpro is computed by fitting all W2 frames simultaneously and incorporating a robust error model. +w2sigp1 %6.3f mag yes Standard deviation of the population of W2 fluxes measured on the w2m individual frames covering this source, in magnitudes. This provides a measure of the characteristic uncertainty of the measurement of this source on individual frames. +w2sigp2 %6.3f mag yes Standard deviation of the mean of the distribution of W2 fluxes (w2magp) computed from profile-fit measurements on the w2m individual frames covering this source, in magnitudes. This is equivalent to w2sigp1/sqrt(w2m). +w3magp %6.3f mag yes Inverse-variance-weighted mean W3 magnitude computed from profile-fit measurements on the w3m individual frames covering this source. This differs from w3mpro in that it is computed by combining the profile-fit measurements from individual frames, whereas w3mpro is computed by fitting all W3 frames simultaneously and incorporating a robust error model. +w3sigp1 %6.3f mag yes Standard deviation of the population of W3 fluxes measured on the w3m individual frames covering this source, in magnitudes. This provides a measure of the characteristic uncertainty of the measurement of this source on individual frames. +w3sigp2 %6.3f mag yes Standard deviation of the mean of the distribution of W3 fluxes (w3magp) computed from profile-fit measurements on the w3m individual frames covering this source, in magnitudes. This is equivalent to w3sigp1/sqrt(w3m). +w4magp %6.3f mag yes Inverse-variance-weighted mean W4 magnitude computed from profile-fit measurements on the w4m individual frames covering this source. This differs from w4mpro in that it is computed by combining the profile-fit measurements from individual frames, whereas w4mpro is computed by fitting all W4 frames simultaneously and incorporating a robust error model. +w4sigp1 %6.3f mag yes Standard deviation of the population of W4 fluxes measured on the w4m individual frames covering this source, in magnitudes. This provides a measure of the characteristic uncertainty of the measurement of this source on individual frames. +w4sigp2 %6.3f mag yes Standard deviation of the mean of the distribution of W4 fluxes (w4magp) computed from profile-fit measurements on the w4m individual frames covering this source, in magnitudes. This is equivalent to w4sigp1/sqrt(w4m). +- 12 fields +xscprox %8.2f arcsec yes 2MASS Extended Source Catalog (XSC) proximity. This column gives the distance between the WISE source position and the position of a nearby 2MASS XSC source, if the separation is less than 1.1 times the Ks isophotal radius size of the XSC source. This identifies WISE sources that are identically the 2MASS XSC source as well as WISE sources that are fragments of large galaxies. This column is "null" if there is no 2MASS XSC source in proximity to the WISE source. +w1rsemi %8.2f arcsec yes Semi-major axis of the elliptical aperture used to measure source in W1. +w1ba %4.2f arcsec yes Axis ratio (b/a) of the elliptical aperture used to measure source in W1. +w1pa %7.2f deg yes Position angle (degrees E of N) of the elliptical aperture major axis used to measure source in W1. +w1gmag %7.3f mag yes W1 magnitude of source measured in the elliptical aperture described by w1rsemi, w1ba, and w1pa. +w1gerr %5.3f mag yes Uncertainty in the W1 magnitude of source measured in elliptical aperture. +w1gflg %1d ---- yes W1 elliptical aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the "standard" aperture photometry quality flag, w1mflg. +w2rsemi %8.2f arcsec yes Semi-major axis of the elliptical aperture used to measure source in W2. +w2ba %4.2f arcsec yes Axis ratio (b/a) of the elliptical aperture used to measure source in W2. +w2pa %7.2f deg yes Position angle (degrees E of N) of the elliptical aperture major axis used to measure source in W2. +w2gmag %7.3f mag yes W2 magnitude of source measured in the elliptical aperture described by w2rsemi, w2ba, and w2pa. +w2gerr %5.3f mag yes Uncertainty in the W2 magnitude of source measured in elliptical aperture. +w2gflg %1d ---- yes W2 elliptical aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the "standard" aperture photometry quality flag, w2mflg. +w3rsemi %8.2f arcsec yes Semi-major axis of the elliptical aperture used to measure source in W3. +w3ba %4.2f arcsec yes Axis ratio (b/a) of the elliptical aperture used to measure source in W3. +w3pa %7.2f deg yes Position angle (degrees E of N) of the elliptical aperture major axis used to measure source in W3. +w3gmag %7.3f mag yes W3 magnitude of source measured in the elliptical aperture described by w3rsemi, w3ba, and w3pa. +w3gerr %5.3f mag yes Uncertainty in the W3 magnitude of source measured in elliptical aperture. +w3gflg %1d ---- yes W3 elliptical aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the "standard" aperture photometry quality flag, w3mflg. +w4rsemi %8.2f arcsec yes Semi-major axis of the elliptical aperture used to measure source in W4. +w4ba %4.2f arcsec yes Axis ratio (b/a) of the elliptical aperture used to measure source in W4. +w4pa %7.2f deg yes Position angle (degrees E of N) of the elliptical aperture major axis used to measure source in W4. +w4gmag %7.3f mag yes W4 magnitude of source measured in the elliptical aperture described by w4rsemi, w4ba, and w4pa. +w4gerr %5.3f mag yes Uncertainty in the W4 magnitude of source measured in elliptical aperture. +w4gflg %1d ---- yes W4 elliptical aperture measurement quality flag. This flag indicates if one or more image pixels in the measurement aperture for this band is confused with nearby objects, is contaminated by saturated or otherwise unusable pixels, or is an upper limit. The flag values are as described for the "standard" aperture photometry quality flag, w4mflg. +- 25 fields +tmass_key %10d ---- yes 2MASS PSC association. Unique identifier of the closest source in the 2MASS Point Source Catalog (PSC) that falls within 3" of the position of this WISE source. This is equivalent to the pts_key in the 2MASS PSC entry. This column is "null" if there is no 2MASS PSC source within 3" of the WISE source position. The number of 2MASS PSC sources within 3" of the WISE position is given in n_2mass +r_2mass %7.3f arcsec yes Distance separating the positions of the WISE source and associated 2MASS PSC source within 3". This column is "null" if there is no associated 2MASS PSC source. +pa_2mass %6.1f deg yes Position angle (degrees E of N) of the vector from the WISE source to the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source. +n_2mass %3d ---- yes The number of 2MASS PSC entries found within a 3" radius of the WISE source position. If more than one 2MASS PSC falls within 3" of the WISE position, the closest 2MASS PSC entry is listed. This column is "null" if there is no associated 2MASS PSC source. +j_m_2mass %7.3f mag yes 2MASS J-band magnitude or magnitude upper limit of the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source or if the 2MASS PSC J-band magnitude entry is "null". +j_msig_2mass %5.3f mag yes 2MASS J-band corrected photometric uncertainty of the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source or if the 2MASS PSC J-band uncertainty entry is "null". +h_m_2mass %7.3f mag yes 2MASS H-band magnitude or magnitude upper limit of the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source or if the 2MASS PSC H-band magnitude entry is "null". +h_msig_2mass %5.3f mag yes 2MASS H-band corrected photometric uncertainty of the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source or if the 2MASS PSC H-band uncertainty entry is "null". +k_m_2mass %7.3f mag yes 2MASS Ks-band magnitude of the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source or if the 2MASS PSC Ks-band magnitude entry is "null". +k_msig_2mass %5.3f mag yes 2MASS Ks-band corrected photometric uncertainty of the associated 2MASS PSC source. This column is "null" if there is no associated 2MASS PSC source or if the 2MASS PSC Ks-band uncertainty entry is "null". +x %19.16f ---- no Unit sphere position x value +y %19.16f ---- no Unit sphere position y value +z %19.16f ---- no Unit sphere position z value +spt_ind %9d ---- no HTM spatial index key Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/doc/wise.sample.dat =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/doc/wise.sample.dat (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/doc/wise.sample.dat (revision 33415) @@ -0,0 +1,3 @@ +J070019.93+432212.0|105.0830765|43.3700066|0.0714|0.5048|-0.0096|173.4548334|19.7690989|101.6541116|20.53902|73.341|623.355|1040143901111000001|1040p439_aa11-000001|1040p439_aa11|1|7.149|0.028|38.9|0.869199991|7.294|0.023|48.1|0.935500026|7.13|0.02|54.8|0.841700017|7.115|0.099|11.0|1.074|0.926599979|1|0|0.142|0.0|0.0|0.0|5000|0000|0|1110|AAAA|15|12|12|12|12|12|12|7|12|10.898|12.111|11.879|12.129|0.0|0.0|0.0|0.0|0||0||0||0||219050.0|5638.0|17.205|3.254|0.0|76247.0|1585.40002|45.53|4.079|0.0|14058.0|256.390015|1881.719|22.688|0.0|225.919998|20.566|736.969|10.674|2.705|7.67|0.005|16|0.222|7.313|0.005|0|0.28|7.118|0.008|0|0.665|7.106|0.162|0|0.616|8.338|0.006|16|8.026|0.006|0|8.369|0.009|0|8.211|0.17|0|7.892|0.005|16|7.593|0.005|0|7.783|0.008|0|7.722|0.162|0|7.725|0.005|16|7.428|0.005|0|7.466|0.007|0|7.414|0.167|1|7.654|0.005|16|7.364|0.005|0|7.3|0.008|0|7.228|0.184|1|7.618|0.005|16|7.332|0.005|0|7.212|0.008|0|7.106|0.21|1|7.596|0.005|16|7.314|0.005|0|7.165|0.009|0|7.052|0.247|1|7.58|0.005|16|7.301|0.005|0|7.139|0.009|0|7.075|0.308|1|7.569|0.005|16|7.291|0.005|0|7.124|0.01|0|7.089|0.375|1|7.147|0.037|0.011|7.295|0.03|0.009|7.13|0.027|0.008|7.129|0.34|0.098||||||||||||||||||||||||||389882849|0.009|81.1|1|8.222|0.034|7.451|0.046|7.231|0.021|-0.18916233986809|0.70189102689035|0.686707066766055|223003002| +J065913.94+431543.9|104.8081131|43.2621994|0.0973|0.5092|-0.012|173.493124|19.5447216|101.4544493|20.4099538|594.029|335.42|1040143901111000051|1040p439_aa11-000051|1040p439_aa11|51|12.893|0.025|42.9|0.7676|12.864|0.03|36.5|0.90079999|12.057||1.7|0.876299977|9.18|||0.902999997|0.857800007|2|0|0.0|0.0|0.0|0.0|0000|0000|0|1100|AAUU|3|13|13|13|13|0|13|0|13|12.628|12.656|12.771|12.717|0.0|0.0|0.0|0.0|0||0||0||0||1103.90002|25.7310009|16.008|3.288|0.0|451.25|12.3649998|44.678|4.026|0.0|69.2180023|40.5629997|1882.381|21.958|0.0|-4.68730021|16.8579998|737.179|10.956|4.392|12.905|0.015|0|0.222|12.907|0.03|0|0.28|11.954||32|0.665|9.494||32|0.616|13.497|0.015|0|13.611|0.031|0|13.002||32|9.6||32|13.127|0.015|0|13.187|0.03|0|12.619||32|10.11||32|12.997|0.016|1|13.048|0.033|1|12.304||32|8.742||32|12.944|0.019|1|13.016|0.04|1|12.015||32|8.286||32|12.914|0.021|1|13.013|0.047|1|11.719||32|7.817||32|12.889|0.024|1|13.007|0.055|1|11.464||32|7.427||32|12.868|0.027|1|12.996|0.063|1|11.264||32|7.166||32|12.853|0.031|1|12.99|0.072|1|11.11||32|7.018||32|12.893|0.023|0.006|12.865|0.068|0.019|12.952|2.605|0.723|||||||||||||||||||||||||||||389862912|0.091|-72.5|1|13.336|0.022|13.016|0.021|13.003|0.028|-0.186121697487002|0.704038676616684|0.685338059319913|223003031| +J065944.84+432452.7|104.9368439|43.4146538|0.1091|0.5117|-0.0212|173.3730353|19.6833296|101.5360666|20.5717047|352.875|736.982|1040143901111000076|1040p439_aa11-000076|1040p439_aa11|76|13.239|0.025|42.8|1.56500006|13.299|0.037|29.1|1.04999995|11.895||1.8|1.01699996|8.748||0.8|0.957199991|1.13900006|1|0|0.0|0.0|0.0|0.0|0000|h000|0|0100|AAUU|3|11|11|11|11|0|11|0|11|11.257|11.534|10.912|11.402|0.0|0.0|0.0|0.0|8|h|0||0||0||802.429993|18.7660007|16.709|3.838|0.0|302.220001|10.3889999|45.571|5.071|0.0|81.6419983|46.5060005|1883.447|23.228|0.0|14.4219999|17.8859997|736.917|10.331|1.413|13.119|0.023|1|0.222|13.132|0.052|1|0.28|12.004|0.508|1|0.665|8.292||32|0.616|13.802|0.025|0|13.954|0.061|0|12.532||32|9.343||32|13.341|0.023|1|13.412|0.052|1|12.669|0.508|1|8.908||32|13.048|0.022|1|13.081|0.049|1|12.099|0.386|1|8.562||32|12.775|0.022|1|12.793|0.047|1|11.694|0.328|1|8.27||32|12.506|0.021|1|12.523|0.044|1|11.434|0.311|1|8.008||32|12.298|0.02|1|12.317|0.043|1|11.252|0.311|1|7.772||32|12.171|0.021|1|12.183|0.044|1|11.152|0.329|1|7.556||32|12.088|0.022|1|12.09|0.047|1|11.163|0.382|1|7.357||32|13.239|0.03|0.009|13.299|0.078|0.023|12.364|0.996|0.3|9.792|2.153|0.649||||||||||||||||||||||||||389863247|0.051|123.7|1|13.972|0.03|13.451|0.031|13.378|0.03|-0.187232347403342|0.701854285923945|0.687273314930839|223003031| Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/include/WISE.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/include/WISE.h (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/include/WISE.h (revision 33415) @@ -0,0 +1,55 @@ + +/* structure for data on a catalog region */ +typedef struct { + char filename[256]; + double RA[2]; + int Nrec; +} WISE_Region; + +typedef struct { + double Rmin, Rmax, Dmin, Dmax; + int index[20]; + int Nindex; +} WISE_Bands; + +typedef struct { + double R, D; + int offset; + int flag; +} WISE_Stars; + +short WISE_W1, WISE_W2, WISE_W3, WISE_W4; + +SkyTable *get2mass_acc (SkyRegion *patch, char *path, char *accel); +Stars *get2mass_2DR_data (SkyRegion *region, char *filename, SkyRegion *patch, int photcode, int *nstars); +Stars *get2mass_AS_data (SkyRegion *region, char *filename, SkyRegion *patch, int phocode, int *nstars); +Stars *get2mass_AS_rawdata (SkyRegion *region, char *filename, SkyRegion *patch, int phocode, int *nstars); + +SkyTable *scan2mass_acc (char *path, char *accel); +int scan2mass_as_data (char *filename); + +char *skipNbounds (char *line, char bound, int Nbound, int Nbyte); +e_time get2mass_time (char *ptr, int Nbound, int Nbyte); +e_time get2mass_date (char *ptr, int Nbound, int Nmax); + +int load2mass_as_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options); +SkyTable *load2mass_acc (char *path, char *accel); +int get2mass_3star (Stars *star, char *line, int Nmax); +int load2mass_catalog (Catalog *catalog, Stars *stars, int Nstars); + +int get2mass_setup (int photcode); +int get2mass_coords (char *line, double *R, double *D, int Nmax); +int get2mass_star (Stars *star, char *line, int Nmax); +int get2mass_3star (Stars *star, char *line, int Nmax); + +int get2mass_3star_full (Stars **star, char *line, int Nmax); +char *next2MASSfield (char *line); +int set2MASS_ph_qual (Stars *star, char qual); +int set2MASS_rd_flag (Stars *star, char qual); +int set2MASS_cc_flag (Stars *star, char qual); +int set2MASS_bl_flag (Stars *star, char qual); +int set2MASS_gal_flag (Stars *star, char qual); +int set2MASS_mp_flag (Stars *star, char qual); +int set2MASS_dup_flag (Stars *star, char qual); +int set2MASS_use_flag (Stars *star, char qual); +int get2mass_sortStars (WISE_Stars *tstars, int Ntstars); Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/src/find_matches_closest.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/src/find_matches_closest.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/src/find_matches_closest.c (revision 33415) @@ -132,4 +132,11 @@ } + /* this block will match a given detection to the closest object within range of that detection. + XXX note that this matches ALL detections within range of the single object to that same object + this is bad, but I cannot just go in linear order (ie, mark off each object as they are + used). I should make a list of all Nobj * Ndet pairs in range and choose the matches + based on their separations. UGH + */ + /* within match range; look for matches */ Jmin = -1; Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise.c (revision 33415) @@ -0,0 +1,52 @@ +# include "addstar.h" +# include "WISE.h" + +int main (int argc, char **argv) { + + char *path; + int i; + SkyTable *sky, *skyWISE; + SkyList *skylist = NULL; + SkyList *overlap = NULL; + AddstarClientOptions options; + + // need to construct these options with args_loadWISE... + options = ConfigInit (&argc, argv); + options = args_loadWISE (argc, argv, options); + + // load the full sky description table: + sky = SkyTableLoadOptimal (CATDIR, SKY_TABLE, GSCFILE, TRUE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // generate the subset matching the user-selected region + skylist = SkyListByPatch (sky, -1, &UserPatch); + + // if we only match to existing (already populated) regions, limit the select to those regions: + if (options.existing_regions) { + SkyList *tmp; + tmp = SkyListExistingSubset (skylist, CATDIR); + SkyListFree (skylist); + skylist = tmp; + } + + path = WISE_DIR_PRELIM; + + // the accel.dat file has the raw filenames + // test if the file exists, or else try the .gz version + // XXX can we build an accelerator? + skyWISE = loadWISE_acc (path, "accel.dat"); + + for (i = 0; i < skyWISE[0].Nregions; i++) { + // check if any of the skylist entries overlap this WISE catalog: + overlap = SkyListByBounds_List (skylist, -1, skyWISE[0].regions[i].Rmin, skyWISE[0].regions[i].Rmax, skyWISE[0].regions[i].Dmin, skyWISE[0].regions[i].Dmax); + if (overlap[0].Nregions == 0) { + SkyListFree (overlap); + continue; + } + + fprintf (stderr, "loading %s\n", skyWISE[0].filename[i]); + loadWISE_as_rawdata (overlap, skyWISE[0].filename[i], options); + SkyListFree (overlap); + } + exit (0); +} Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_ops.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_ops.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_ops.c (revision 33415) @@ -0,0 +1,155 @@ +# include "addstar.h" +# include "WISE.h" + +static int FilterSkip; +static int TimeSkip; +static int Qentry; +static int Photcode; + +int getWISE_setup () { + + NAMED_PHOTCODE (WISE_W1, "WISE_W1"); + NAMED_PHOTCODE (WISE_W2, "WISE_W2"); + NAMED_PHOTCODE (WISE_W3, "WISE_W3"); + NAMED_PHOTCODE (WISE_W4, "WISE_W4"); + + return TRUE; +} + +// fill in the coords for a single star. takes a pointer to the start of the line +int getWISE_coords (char *line, double *R, double *D, int Nmax) { + + char *ptr = line; + + ptr = skipNbounds (ptr, '|', 1, Nmax); + *R = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 1, Nmax); + *D = strtod (ptr, NULL); + if (*D > 90) Shutdown ("weird DEC value: something is wrong"); + + return TRUE; +} + +int getWISE_sortStars (WISE_Stars *tstars, int Ntstars) { + +# define SWAPFUNC(A,B){ WISE_Stars temp = tstars[A]; tstars[A] = tstars[B]; tstars[B] = temp; } +# define COMPARE(A,B)(tstars[A].R < tstars[B].R) + + OHANA_SORT (Ntstars, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + + return TRUE; +} + +// this function retrieves the time from the DATE field +e_time getWISE_date (char *ptr, int Nbound, int Nmax) { + + e_time time; + char *p, *end; + + p = skipNbounds (ptr, '|', Nbound, Nmax); + if (p == NULL) Shutdown ("format error in WISE"); + end = memchr (p, '|', Nmax - (p - ptr)); + if (end == NULL) Shutdown ("format error in WISE"); + *end = 0; + time = ohana_date_to_sec (ptr); + *end = '|'; + + return (time); +} + +// this function retrieves the time from the JDATE field (%12.4f) +e_time getWISE_time (char *ptr, int Nbound, int Nmax) { + + e_time time; + double jd; + char *p, *end; + + p = skipNbounds (ptr, '|', Nbound, Nmax); + if (p == NULL) Shutdown ("format error in WISE"); + end = memchr (p, '|', Nmax - (p - ptr)); + if (end == NULL) Shutdown ("format error in WISE"); + *end = 0; + jd = strtod (p, NULL); + time = ohana_jd_to_sec (jd); + *end = '|'; + + return (time); +} + +/* return a pointer to the first char after Nbound of value bound */ +char *skipNbounds (char *line, char bound, int Nbound, int Nbyte) { + + int i; + char *p, *q; + + p = line; + for (i = 0; i < Nbound; i++) { + q = memchr (p, bound, Nbyte - (p - line)); + if (q == NULL) return (NULL); + p = q + 1; + if (p - line == Nbyte) return (NULL); + } + return (p); +} + +/* watch for patches which cross 0,360 boundary */ +SkyTable *loadWISE_acc (char *path, char *accel) { + + int Nregions, NREGIONS, Nrec; + char accelfile[1024], line[256], filename[128], datafile[256], **filenames; + FILE *f; + double Rs, Re, Ds, De; + + SkyTable *sky; + SkyRegion *regions; + + sprintf (accelfile, "%s/%s", path, accel); + f = fopen (accelfile, "r"); + if (f == NULL) Shutdown ("can't read data from accelerator %s", accelfile); + + Nregions = 0; + NREGIONS = 200; + ALLOCATE (regions, SkyRegion, NREGIONS); + ALLOCATE (filenames, char *, NREGIONS); + + /* read in stars line-by-line */ + while (scan_line (f, line) != EOF) { + stripwhite (line); + if (line[0] == 0) continue; + if (line[0] == '#') continue; + sscanf (line, "%s %lf %lf %lf %lf %d", filename, &Rs, &Re, &Ds, &De, &Nrec); + Rs *= 15.0; + Re *= 15.0; + + // don't restrict by RA, but limit by DEC + if (De < UserPatch.Dmin) continue; + if (Ds > UserPatch.Dmax) continue; + + regions[Nregions].Rmin = Rs; + regions[Nregions].Rmax = Re; + regions[Nregions].Dmin = Ds; + regions[Nregions].Dmax = De; + regions[Nregions].childE = Nrec; // a cheat since WISE only has one depth + + sprintf (datafile, "%s/%s", path, filename); + filenames[Nregions] = strcreate (datafile); + + Nregions ++; + if (Nregions >= NREGIONS) { + NREGIONS += 20; + REALLOCATE (regions, SkyRegion, NREGIONS); + REALLOCATE (filenames, char *, NREGIONS); + } + } + fclose (f); + + ALLOCATE (sky, SkyTable, 1); + sky[0].regions = regions; + sky[0].filename = filenames; + sky[0].Nregions = Nregions; + return (sky); +} + Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_prelim_rawdata.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_prelim_rawdata.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_prelim_rawdata.c (revision 33415) @@ -0,0 +1,241 @@ +# include "addstar.h" +# include "WISE.h" + +/* unlike the DR2 data, the AS data is NOT fixed bytes/row + * we need to handle fractional lines at the end of each read block + */ + +/* read in chunks of ~64MB */ +# define NBYTE 0x4000000 +# define DEBUG 0 + +int loadWISE_as_rawdata (SkyList *skytable, char *filename, AddstarClientOptions options) { + + int i, j, verbose; + int Nstars, NSTARS, Ntstars, NTSTARS; + int Nbyte, Nextra, Ntotal, offset; + + double Rmin, Rmax, Dmin, Dmax; + + FILE *f; + char *buffer, *p, *q; + + Stars **stars; // this is an array of pointers to be consistent with input to find_match_refstars + WISE_Stars *tstars; + SkyList *skylist; + SkyRegion *region; + Catalog catalog; + + getWISE_setup (); + + ALLOCATE (buffer, char, NBYTE); + + // scan through the entire WISE file + f = fopen (filename, "r"); + if (f == NULL) Shutdown ("can't read WISE data file: %s", filename); + // test if this is a raw datafile or gzipped... + + Nextra = 0; // number excess bytes from lsat partial row + Ntotal = 0; // track the total number of bytes read + while ((Nbyte = fread (&buffer[Nextra], 1, NBYTE-Nextra, f)) != 0) { + if (Nbyte == -1) Shutdown ("error reading from raw file %s", filename); + if (DEBUG) fprintf (stderr, "read %d bytes", Nbyte); + + Nbyte += Nextra; + + if (VERBOSE) fprintf (stderr, "read .. "); + + /* find bounds on first complete line */ + p = buffer; + q = memchr (p, '\n', Nbyte); + if (q == NULL) Shutdown ("incomplete line at end of file\n"); + offset = p - buffer; // offset within this scan + + Ntstars = 0; + NTSTARS = 10000; + ALLOCATE (tstars, WISE_Stars, NTSTARS); + + Rmin = 360.0; + Rmax = 0.0; + Dmin = +90.0; + Dmax = -90.0; + + // scan through entire buffer for star coords + while (1) { + getWISE_coords (p, &tstars[Ntstars].R, &tstars[Ntstars].D, Nbyte - offset); + tstars[Ntstars].offset = offset; // offset within scan + tstars[Ntstars].flag = FALSE; + + if (VERBOSE) { + Rmin = MIN (Rmin, tstars[Ntstars].R); + Rmax = MAX (Rmax, tstars[Ntstars].R); + Dmin = MIN (Dmin, tstars[Ntstars].D); + Dmax = MAX (Dmax, tstars[Ntstars].D); + } + + Ntstars ++; + CHECK_REALLOCATE (tstars, WISE_Stars, NTSTARS, Ntstars, 10000); + + /* start of the next line */ + p = q + 1; + offset = p - buffer; // offset within this scan + if (offset == Nbyte) { + // last line in buffer is a complete line + Nextra = 0; + break; + } + /* end of the next line */ + q = memchr (p, '\n', Nbyte - offset); + if (q == NULL) { + // last, incomplete line in buffer + Nextra = Nbyte - offset; + break; + } + } + if (VERBOSE) fprintf (stderr, "scan %d stars (%10.6f - %10.6f, %10.6f - %10.6f) .. ", Ntstars, Rmin, Rmax, Dmin, Dmax); + + // sort the tstars by RA + getWISE_sortStars (tstars, Ntstars); + + // scan through the stars, loading the containing catalogs + // skip through table for unsaved stars + for (i = 0; i < Ntstars; i++) { + if (tstars[i].flag) continue; + + // scan forward until we read the UserPatch + if (tstars[i].R < UserPatch.Rmin) continue; + if (tstars[i].R > UserPatch.Rmax) break; + if (tstars[i].D < UserPatch.Dmin) continue; + if (tstars[i].D > UserPatch.Dmax) continue; + + // identify the relevant catalog + skylist = SkyRegionByPoint_List (skytable, -1, tstars[i].R, tstars[i].D); + if (skylist[0].Nregions == 0) { + SkyListFree (skylist); + continue; + } + region = skylist[0].regions[0]; + if (DEBUG) fprintf (stderr, "writing to %s\n", skylist[0].filename[0]); + + // collect array of (Stars *) stars in a new output catalog + Nstars = 0; + NSTARS = 3000; + ALLOCATE (stars, Stars *, NSTARS); + + // loop over stars in this WISE region that are also in this output region + for (j = i; j < Ntstars; j++) { + if (tstars[j].flag) continue; + + // check if in skyregion + if (tstars[j].R < region[0].Rmin) continue; + if (tstars[j].R > region[0].Rmax) break; + if (tstars[j].D < region[0].Dmin) continue; + if (tstars[j].D > region[0].Dmax) continue; + + // check if in UserPatch + if (tstars[j].R < UserPatch.Rmin) continue; + if (tstars[j].R > UserPatch.Rmax) break; + if (tstars[j].D < UserPatch.Dmin) continue; + if (tstars[j].D > UserPatch.Dmax) continue; + + offset = tstars[j].offset; + + ALLOCATE (stars[Nstars+0], Stars, 1); + ALLOCATE (stars[Nstars+1], Stars, 1); + ALLOCATE (stars[Nstars+2], Stars, 1); + ALLOCATE (stars[Nstars+3], Stars, 1); + + InitStar (stars[Nstars+0]); + InitStar (stars[Nstars+1]); + InitStar (stars[Nstars+2]); + InitStar (stars[Nstars+3]); + + stars[Nstars+0][0].average.R = tstars[j].R; + stars[Nstars+0][0].average.D = tstars[j].D; + stars[Nstars+1][0].average.R = tstars[j].R; + stars[Nstars+1][0].average.D = tstars[j].D; + stars[Nstars+2][0].average.R = tstars[j].R; + stars[Nstars+2][0].average.D = tstars[j].D; + stars[Nstars+3][0].average.R = tstars[j].R; + stars[Nstars+3][0].average.D = tstars[j].D; + getWISE_star_full (&stars[Nstars], &buffer[offset], Nbyte - offset); + + tstars[j].flag = TRUE; + + Nstars += 4; + if (Nstars >= NSTARS - 4) { + NSTARS += 4000; + REALLOCATE (stars, Stars *, NSTARS); + } + } + + if (!Nstars) { + free (stars); + continue; + } + + if (DEBUG) fprintf (stderr, "selected %d stars (%10.6f - %10.6f, %10.6f - %10.6f)\n", Nstars, + region[0].Rmin, region[0].Rmax, region[0].Dmin, region[0].Dmax); + + if (1) { + verbose = VERBOSE; + VERBOSE = FALSE; + + // now we have all of the loaded stars in this catalog + catalog.filename = skylist[0].filename[0]; + catalog.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog.Nsecfilt = GetPhotcodeNsecfilt (); + + // an error exit status here is a significant error + if (!dvo_catalog_open (&catalog, skylist[0].regions[0], VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure to open/create catalog file %s\n", catalog.filename); + exit (2); + } + + // assume no input star matches an existing star + // simply add to the existing table + // loadWISE_catalog (&catalog, stars, Nstars); + find_matches_refstars (skylist[0].regions[0], stars, Nstars, &catalog, options); + // loadWISE_catalog (&catalog, stars, Nstars); + + dvo_catalog_save (&catalog, VERBOSE); + dvo_catalog_unlock (&catalog); + dvo_catalog_free (&catalog); + // free (catalog.filename); + // XXX don't free this! it points to an element of the skytable + } + + SkyListFree (skylist); + for (j = 0; j < Nstars; j++) free (stars[j]); + free (stars); + VERBOSE = verbose; + } + free (tstars); + if (VERBOSE) fprintf (stderr, "done\n"); + + // at end, p points at the start of last, partial line + if (Nextra) memmove (buffer, p, Nextra); + } + + if (VERBOSE) fprintf (stderr, "\n"); + + fclose (f); + free (buffer); + return (TRUE); +} + +/* + for each WISE file: + for each data block + generate a table of: R, D, byte, flag + for each unsaved star + find containing catalog + load catalog + find all contained stars + add to catalog + save catalog + mark all contained stars +*/ + Index: branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_star_full.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_star_full.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/addstar/src/loadwise_star_full.c (revision 33415) @@ -0,0 +1,264 @@ +# include "addstar.h" +# include "WISE.h" + +// XXX check to see if desired output format is PS1_V1 or later? (use 16bit version if not?) + +// fill in the data for a WISE quad star. takes a pointer to the start of the line the +// RA and DEC have already been set +int getWISE_star_full (Stars **star, char *line, int Nmax) { + + char *ptr; + double dMfull; + double jd; + + if (line == NULL) Shutdown ("format error in WISE"); + + ptr = skipNbounds (ptr, '|', 3, Nmax); // skip: desig, ra, dec, + star[0][0].measure.FWx = ToShortPixels(strtod (ptr, NULL)); + ptr = nextWISEfield (ptr); // sig_dec + star[0][0].measure.FWy = ToShortPixels(strtod (ptr, NULL)); + ptr = nextWISEfield (ptr); // sig_radec + star[0][0].measure.theta = ToShortPixels(strtod (ptr, NULL)); + + star[1][0].measure.FWx = star[0][0].measure.FWx; + star[1][0].measure.FWy = star[0][0].measure.FWy; + star[1][0].measure.theta = star[0][0].measure.theta; + star[2][0].measure.FWx = star[0][0].measure.FWx; + star[2][0].measure.FWy = star[0][0].measure.FWy; + star[2][0].measure.theta = star[0][0].measure.theta; + star[3][0].measure.FWx = star[0][0].measure.FWx; + star[3][0].measure.FWy = star[0][0].measure.FWy; + star[3][0].measure.theta = star[0][0].measure.theta; + + ptr = skipNbounds (ptr, '|', 10, Nmax); // skip: glon, glat, elon, elat, wx, wy, cntr, source_id, coadd_id, src + + // W1 + star[0][0].measure.M = strtod (ptr, NULL); + ptr = nextWISEfield (ptr); // j_cmsig + star[0][0].measure.dM = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: w1snr, w1rchi2 + + // W2 + star[1][0].measure.M = strtod (ptr, NULL); + ptr = nextWISEfield (ptr); // j_cmsig + star[1][0].measure.dM = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: w2snr, w2rchi2 + + // W3 + star[2][0].measure.M = strtod (ptr, NULL); + ptr = nextWISEfield (ptr); // j_cmsig + star[2][0].measure.dM = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: w3snr, w3rchi2 + + // W4 + star[3][0].measure.M = strtod (ptr, NULL); + ptr = nextWISEfield (ptr); // j_cmsig + star[3][0].measure.dM = strtod (ptr, NULL); + ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: w4snr, w4rchi2 + + star[0][0].measure.photFlags = 0; + star[1][0].measure.photFlags = 0; + star[2][0].measure.photFlags = 0; + star[3][0].measure.photFlags = 0; + ptr = skipNbounds (ptr, '|', 8, Nmax); // skip fields + + setWISE_cc_flag (star[0], ptr[0]); // cc_flg + setWISE_cc_flag (star[1], ptr[1]); // cc_flg + setWISE_cc_flag (star[2], ptr[2]); // cc_flg + setWISE_cc_flag (star[3], ptr[2]); // cc_flg + ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: ext_flg, var_flg + + setWISE_ph_qual (star[0], ptr[0]); // ph_qual + setWISE_ph_qual (star[1], ptr[1]); // ph_qual + setWISE_ph_qual (star[2], ptr[2]); // ph_qual + setWISE_ph_qual (star[3], ptr[3]); // ph_qual + ptr = skipNbounds (ptr, '|', 2, Nmax); // skip: ext_flg, var_flg + + // setWISE_rd_flag (star[0], ptr[0]); + // setWISE_bl_flag (star[0], ptr[0]); + // setWISE_gal_flag (star[0], ptr[0]); + // setWISE_mp_flag (star[0], ptr[0]); + + // ptr = nextWISEfield (ptr); // x_scan + // star[0][0].measure.Xccd = strtod (ptr, NULL); + // star[2][0].measure.Xccd = star[1][0].measure.Xccd = star[0][0].measure.Xccd; + // ptr = nextWISEfield (ptr); // jdate (julian date) + + jd = 2455263.0; // NOTE : WISE prelim release does not contain per-detection time info. + // the release is based on data taken in the period 14 January 2010 to 29 April 2010 + star[0][0].measure.t = ohana_jd_to_sec (jd); + star[1][0].measure.t = star[0][0].measure.t; + star[2][0].measure.t = star[0][0].measure.t; + star[3][0].measure.t = star[0][0].measure.t; + + // ptr = nextWISEfield (ptr); // j_psfchi + // star[0][0].measure.psfChisq = strtod (ptr, NULL); + // ptr = nextWISEfield (ptr); // h_psfchi + // star[1][0].measure.psfChisq = strtod (ptr, NULL); + // ptr = nextWISEfield (ptr); // k_psfchi + // star[2][0].measure.psfChisq = strtod (ptr, NULL); + // + // ptr = nextWISEfield (ptr); // j_m_stdap + // star[0][0].measure.Map = strtod (ptr, NULL); + // ptr = nextWISEfield (ptr); // j_msig_stdap (skip?) + // + // ptr = nextWISEfield (ptr); // h_m_stdap + // star[1][0].measure.Map = strtod (ptr, NULL); + // ptr = nextWISEfield (ptr); // h_msig_stdap (skip?) + // + // ptr = nextWISEfield (ptr); // k_m_stdap + // star[2][0].measure.Map = strtod (ptr, NULL); + // ptr = nextWISEfield (ptr); // k_msig_stdap (skip?) + // + // ptr = nextWISEfield (ptr); // dist_edge_ns (skip) + // ptr = nextWISEfield (ptr); // dist_edge_ew (skip) + // ptr = nextWISEfield (ptr); // dist_edge_flg (skip) + // + // ptr = nextWISEfield (ptr); // dup_src (one flag for all filters) + // setWISE_dup_flag (star[0], ptr[0]); + // setWISE_dup_flag (star[1], ptr[0]); + // setWISE_dup_flag (star[2], ptr[0]); + // + // ptr = nextWISEfield (ptr); // use_src (one flag for all filters) + // setWISE_use_flag (star[0], ptr[0]); + // setWISE_use_flag (star[1], ptr[0]); + // setWISE_use_flag (star[2], ptr[0]); + + star[0][0].measure.photcode = TM_J; + star[0][0].measure.detID = 0; + star[0][0].measure.imageID = 0; + + star[1][0].measure.photcode = TM_H; + star[1][0].measure.detID = 0; + star[1][0].measure.imageID = 0; + + star[2][0].measure.photcode = TM_K; + star[2][0].measure.detID = 0; + star[2][0].measure.imageID = 0; + + return TRUE; +} + +/* return a pointer to the first char after the next field separator (|) */ +char *nextWISEfield (char *line) { + + char *p, *q; + + p = line; + q = strchr (p, '|'); + if (q == NULL) return (NULL); + p = q + 1; + if (*p == 0) return (NULL); + return (p); +} + +int setWISE_cc_flag (Stars *star, char qual) { + + switch (qual) { + case 'p': star[0].measure.photFlags |= 0x00010000; break; + case 'P': star[0].measure.photFlags |= 0x00010000; break; + case 'h': star[0].measure.photFlags |= 0x00020000; break; + case 'H': star[0].measure.photFlags |= 0x00020000; break; + case 'd': star[0].measure.photFlags |= 0x00040000; break; + case 'D': star[0].measure.photFlags |= 0x00040000; break; + case 'o': star[0].measure.photFlags |= 0x00020000; break; + case 'O': star[0].measure.photFlags |= 0x00020000; break; + case '0': break; + default: + fprintf (stderr, "error!\n"); + exit (2); + } + return (TRUE); +} + +int setWISE_ph_qual (Stars *star, char qual) { + + switch (qual) { + case 'A': star[0].measure.photFlags |= 0x00000001; break; // was: 0x0004 + case 'B': star[0].measure.photFlags |= 0x00000002; break; // was: 0x0005 + case 'C': star[0].measure.photFlags |= 0x00000004; break; // was: 0x0006 + case 'U': star[0].measure.photFlags |= 0x00000040; break; // was: 0x0001 + case 'X': star[0].measure.photFlags |= 0x00000080; break; // was: 0x0000 + default: + fprintf (stderr, "error!\n"); + exit (2); + } + return (TRUE); +} + +int setWISE_rd_flag (Stars *star, char qual) { + + switch (qual) { + case '0': star[0].measure.photFlags |= 0x00000100; break; // was: 0x0000 + case '1': star[0].measure.photFlags |= 0x00000200; break; // was: 0x0010 + case '2': star[0].measure.photFlags |= 0x00000400; break; // was: 0x0020 + case '3': star[0].measure.photFlags |= 0x00000800; break; // was: 0x0030 + case '4': star[0].measure.photFlags |= 0x00001000; break; // was: 0x0040 + case '6': star[0].measure.photFlags |= 0x00002000; break; // was: 0x0050 + case '9': star[0].measure.photFlags |= 0x00004000; break; // was: 0x0060 + default: + fprintf (stderr, "error!\n"); + exit (2); + } + return (TRUE); +} + +int setWISE_bl_flag (Stars *star, char qual) { + + switch (qual) { + case '0': star[0].measure.photFlags &= ~0x00300000; break; // was: ~0x0008 + case '1': star[0].measure.photFlags |= 0x00100000; break; // was: ~0x0008 + default: star[0].measure.photFlags |= 0x00200000; break; // was: 0x0008 + } + return (TRUE); +} + +int setWISE_gal_flag (Stars *star, char qual) { + + switch (qual) { + case '0': star[0].measure.photFlags &= ~0x00c00000; break; // was: ~0x0080 + case '1': star[0].measure.photFlags |= 0x00400000; break; // was: ~0x0080 + default: star[0].measure.photFlags |= 0x00800000; // was: 0x0080 + star[0].measure.extNsigma = 100.0; + break; + } + return (TRUE); +} + +int setWISE_mp_flag (Stars *star, char qual) { + + switch (qual) { + case '0': star[0].measure.photFlags &= ~0x03000000; break; // was: ~0x0800 + case '1': star[0].measure.photFlags |= 0x01000000; break; // was: ~0x0800 + default: star[0].measure.photFlags |= 0x02000000; break; // was: 0x0800 + } + return (TRUE); +} + +int setWISE_dup_flag (Stars *star, char qual) { + + switch (qual) { + case '0': star[0].measure.photFlags &= ~0x0c000000; break; // was: ~0x1000 + case '1': star[0].measure.photFlags |= 0x04000000; break; // was: ~0x1000 + default: star[0].measure.photFlags |= 0x08000000; break; // was: 0x1000 + } + return (TRUE); +} + +int setWISE_use_flag (Stars *star, char qual) { + + switch (qual) { + case '0': star[0].measure.photFlags &= ~0x10000000; break; // was: ~0x2000 + case '1': star[0].measure.photFlags |= 0x10000000; break; // was: 0x2000 + default: abort(); + } + return (TRUE); +} + +// unused photFlags: +// 0x0000.8000 +// 0x0004.0000 +// 0x0008.0000 +// 0x2000.0000 +// 0x4000.0000 +// 0x8000.0000 Index: branches/meh_branches/ppstack_test/Ohana/src/getstar/src/dvoImagesAtCoords.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/getstar/src/dvoImagesAtCoords.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/getstar/src/dvoImagesAtCoords.c (revision 33415) @@ -158,5 +158,7 @@ } if (copy) { - free(copy); + // XXX: Temporarily leak this copy. free gets redirected to + // ohana_free but strdup didn't allocate the memory + // free(copy); } } Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/include/prototypes.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/include/prototypes.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/include/prototypes.h (revision 33415) @@ -126,4 +126,5 @@ void PSPixmap24 PROTO((Graphic *graphic, KapaImageWidget *image, FILE *f)); void PSPixmap32 PROTO((Graphic *graphic, KapaImageWidget *image, FILE *f)); +void PSPixmap_3byte PROTO((Graphic *graphic, KapaImageWidget *image, FILE *f)); /* kapa bDraw Functions */ Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/include/structures.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/include/structures.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/include/structures.h (revision 33415) @@ -26,6 +26,8 @@ XFontStruct *font; Cursor cursor; - int x, y; - unsigned int dx, dy; + int x, y; // corner coord in X world coords + unsigned int dx, dy; // size of window in X coords + int xwin, ywin; // corner coord of display subregion (eg, png, ps plot) + int dxwin, dywin; // corner coord of display subregion (eg, png, ps plot) CCNode *cube; XColor *cmap; Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/DrawFrame.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/DrawFrame.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/DrawFrame.c (revision 33415) @@ -150,5 +150,6 @@ } -# define MIN_RANGE 1e-10 +// do I really want this? allow an override? +# define MIN_RANGE 1e-30 void AxisTickScale (Axis *axis, double *major, double *minor, int *nsignif) { Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/DrawObjects.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/DrawObjects.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/DrawObjects.c (revision 33415) @@ -408,4 +408,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -424,4 +425,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -440,4 +442,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -457,4 +460,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -475,4 +479,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -495,4 +500,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -513,4 +519,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -531,4 +538,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -547,4 +555,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -567,4 +576,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); @@ -589,4 +599,5 @@ (sy < graph[0].axis[1].fy) && (sy > graph[0].axis[1].fy + graph[0].axis[1].dfy)) { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); XSetForeground (graphic->display, graphic->gc, graphic->cmap[pixel].pixel); Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSObjects.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSObjects.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSObjects.c (revision 33415) @@ -323,4 +323,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -340,4 +341,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -357,4 +359,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -375,4 +378,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -393,4 +397,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -415,4 +420,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -434,4 +440,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -453,4 +460,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -470,4 +478,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -491,4 +500,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); @@ -514,4 +524,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); fprintf (f, "%4.2f %4.2f %4.2f setrgbcolor\n", pixel1[pixel], pixel2[pixel], pixel3[pixel]); Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSPixmap.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSPixmap.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSPixmap.c (revision 33415) @@ -1,5 +1,3 @@ # include "Ximage.h" - -// XXX this stuff has been broken by the conversion to the pixmap stuff void PSPixmap8 (Graphic *graphic, KapaImageWidget *image, FILE *f) { @@ -146,40 +144,127 @@ } -# if (0) -// XXX needs work! -void PSPixmap32_RGB (Graphic *graphic, KapaImageWidget *image, FILE *f) { - - int i, k, m, val; - double Nchar, Npix, start, slope, frac; - unsigned int *buff; - unsigned long back; - unsigned char - - ALLOCATE (pixelR, unsigned char, graphic[0].Npixels); - ALLOCATE (pixelG, unsigned char, graphic[0].Npixels); - ALLOCATE (pixelB, unsigned char, graphic[0].Npixels); +# define WHITE_R 255 +# define WHITE_G 255 +# define WHITE_B 255 + +void PSPixmap_3byte (Graphic *graphic, KapaImageWidget *image, FILE *f) { + + int i, j, ii, jj; + int i_start, i_end, j_start, j_end; + int I_start, J_start; + int dropback; /* this is a bit of a kludge... */ + int dx, dy, DX, DY, inDX, inDY, Xs, Ys; + int expand_in, expand_out; + double expand, Ix, Iy; + unsigned short *in_pix, *in_pix_ref; + unsigned char *pixel1, *pixel2, *pixel3; + + if (image == NULL) return; + + ALLOCATE (pixel1, unsigned char, graphic[0].Npixels); + ALLOCATE (pixel2, unsigned char, graphic[0].Npixels); + ALLOCATE (pixel3, unsigned char, graphic[0].Npixels); /** cmap[i].pixel must be defined even if X is not used **/ for (i = 0; i < graphic[0].Npixels; i++) { /* set up pixel array */ - pixelR[i] = graphic[0].cmap[i].red >> 8; - pixelG[i] = graphic[0].cmap[i].green >> 8; - pixelB[i] = graphic[0].cmap[i].blue >> 8; - } - - for (i = 0; i < image[0].picture.dy; i++) { - for (k = 0; k < image[0].picture.dx; k++, buff++) { - if (*buff == back) - val = Nchar; - else { - for (m = 0; (graphic[0].cmap[m].pixel != *buff) && (m < Npix); m++); - val = Nchar - frac * MIN (MAX (start + m * slope, 0), Npix); - } - fprintf (f, "%02x", val); - if (!((k+1) % 40)) fprintf (f, "\n"); - } - fprintf (f, "\n"); - buff -= 2*image[0].picture.dx; - } - return; -} -# endif + pixel1[i] = graphic[0].cmap[i].red >> 8; + pixel2[i] = graphic[0].cmap[i].green >> 8; + pixel3[i] = graphic[0].cmap[i].blue >> 8; + } + + assert ((image[0].picture.expand >= 1) || (image[0].picture.expand <= -2)); + expand = expand_in = expand_out = 1.0; + if (image[0].picture.expand > 0) { + expand = 1 / (1.0*image[0].picture.expand); + expand_out = image[0].picture.expand; + expand_in = 1; + } + if (image[0].picture.expand < 0) { + expand = fabs((double)image[0].picture.expand); + expand_out = 1; + expand_in = -image[0].picture.expand; + } + + Xs = image[0].picture.x; + Ys = image[0].picture.y; + dx = image[0].picture.dx; + dy = image[0].picture.dy; + DX = image[0].image[0].matrix.Naxis[0]; + DY = image[0].image[0].matrix.Naxis[1]; + + // i_start, j_start are the closest lit screen pixel to 0,0 + // I_start, J_start are the image pixel corresponding to i_start, j_start + Picture_Lower (&i_start, &j_start, &I_start, &J_start, &image[0].image[0].matrix, &image[0].picture); + + // i_end, j_end are the closest lit screen pixel to dx, dy + // I_end, J_end are the image pixel corresponding to i_end, j_end + Picture_Upper (&i_end, &j_end, i_start, j_start, &image[0].image[0].matrix, &image[0].picture); + + assert (i_start <= i_end); + assert (j_start <= j_end); + + Ix = image[0].picture.flipx ? I_start - 1 : I_start; + Iy = image[0].picture.flipy ? J_start - 1 : J_start; + + inDX = image[0].picture.flipx ? -1 : +1; + inDY = image[0].picture.flipy ? -1 : +1; + + dropback = expand_out - (i_end - i_start) % expand_out; + if ((i_end - i_start) % expand_out == 0) dropback = 0; + + in_pix_ref = &image[0].pixmap[DX*(int)MAX(Iy,0) + (int)MAX(Ix,0)]; + + /********** below we do the mapping from buffer pixels (in) to picture pixels (out) **********/ + + // add in occasional return chars + + /**** fill in bottom area ****/ + for (j = 0; j < j_start; j++) { + for (i = 0; i < dx; i++) { + fprintf (f, "%02x%02x%02x", WHITE_R, WHITE_G, WHITE_B); + } + } + + // probably could do this all smarter with scale operations in PS... + + /*** fill in the image data region ***/ + for (j = j_start; j < j_end; j+= expand_out, in_pix_ref += inDY*expand_in*DX) { + + // repeat the section below 'expand_out' times + for (jj = 0; jj < expand_out; jj++) { + + /* create one output image line */ + in_pix = in_pix_ref; + + /**** fill in area to the left of the picture ****/ + for (i = 0; i < i_start; i++) { + fprintf (f, "%02x%02x%02x", WHITE_R, WHITE_G, WHITE_B); + } + + /*** fill in the picture region ***/ + for (i = i_start; i < i_end; i+=expand_out, in_pix += inDX*expand_in) { + for (ii = 0; ii < expand_out; ii++) { + fprintf (f, "%02x%02x%02x", pixel1[*in_pix], pixel2[*in_pix], pixel3[*in_pix]); + } + } + + /**** fill in area to the right of the picture ****/ + for (i = i_end; i < dx; i++) { + fprintf (f, "%02x%02x%02x", WHITE_R, WHITE_G, WHITE_B); + } + } + } + + /**** fill in top area ****/ + for (j = j_end; j < dy; j++) { + for (i = 0; i < dx; i++) { + fprintf (f, "%02x%02x%02x", WHITE_R, WHITE_G, WHITE_B); + } + } + + free (pixel1); + free (pixel2); + free (pixel3); + + return; +} Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSimage.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSimage.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/PSimage.c (revision 33415) @@ -13,4 +13,11 @@ graphic = GetGraphic(); + // Update this to generate color PS images: I need to test if the image if color or BW (based + // on the colormap). If it is a color image, I need to generate a 24 bit image. the header + // should look the same (Nx Ny 8 [1 0 0 1 0 0]) but then it should finish with "false 3 + // colorimage" instead of just "image" (see ../doc/color.image.s for an example). the hex + // string needs to have RGB represented as chars (2 hex chars per R,G,B). + // the values come from the following map: image[0].pixmap[i] -> cmap[pixel].red -> R (0-255) + fprintf (f, " newpath %d %d moveto %d %d lineto %d %d lineto %d %d lineto closepath\n\n", (int) image[0].picture.x, graphic->dy - (int) image[0].picture.y, @@ -21,23 +28,18 @@ fprintf (f, "%d %d translate\n", (int) image[0].picture.x, graphic->dy - (int) image[0].picture.y - image[0].picture.dy); fprintf (f, "%d %d 8\n", image[0].picture.dx, image[0].picture.dy); - fprintf (f, "[1 0 0 1 0 0]\n"); + fprintf (f, "[1 0 0 -1 0 %d]\n", image[0].picture.dy); + // write out the image in normal order, but flip in PS + +# if (0) + fprintf (f, "{currentfile %d string readhexstring pop} image\n\n", image[0].picture.dx); + PSPixmap_1byte (graphic, image, f); - /******** First we draw the picture itself ********/ - /* in !USE_XWINDOW, we'll have to change this to use the JPEG function */ - switch (graphic[0].Nbits) { - case 8: - PSPixmap8 (graphic, image, f); - break; - case 16: - PSPixmap16 (graphic, image, f); - break; - case 24: - PSPixmap24 (graphic, image, f); - break; - case 32: - PSPixmap32 (graphic, image, f); - break; - } +# else + + fprintf (f, "{currentfile %d string readhexstring pop} false 3 colorimage\n\n", 3*image[0].picture.dx); + PSPixmap_3byte (graphic, image, f); + +# endif fprintf (f, "grestore %% end of image\n"); Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/Sections.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/Sections.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/Sections.c (revision 33415) @@ -237,2 +237,35 @@ return (TRUE); } + +int SectionMinBoundary (Graphic *graphic) { + + int i; + int Xs = graphic->dx; + int Ys = graphic->dy; + int Xe = 0; + int Ye = 0; + + // the boundary for a single section should probably be adjusted depending on the + // image status (should not include the imtool portion) + for (i = 0; i < Nsections; i++) { + Xs = MIN (Xs, sections[i][0].x); + Ys = MIN (Ys, sections[i][0].y); + Xe = MAX (Xe, sections[i][0].x + sections[i][0].dx); + Ye = MAX (Ye, sections[i][0].y + sections[i][0].dy); + } + + if ((Xs >= Xe) || (Ys >- Ye)) { + // default values for the region window + graphic->xwin = 0; + graphic->ywin = 0; + graphic->dxwin = graphic->dx; + graphic->dywin = graphic->dy; + } + + // set min/max boundary (min window containing max range of sections) + graphic->xwin = Xs; + graphic->ywin = Ys; + graphic->dxwin = Xe - Xs; + graphic->dywin = Ye - Ys; + return TRUE; +} Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/SetUpGraphic.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/SetUpGraphic.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/SetUpGraphic.c (revision 33415) @@ -16,4 +16,10 @@ graphic->dx = 512; graphic->dy = 512; + + // default values for the region window + graphic->xwin = 0; + graphic->ywin = 0; + graphic->dxwin = graphic->dx; + graphic->dywin = graphic->dy; if (!USE_XWINDOW) { Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawImage.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawImage.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawImage.c (revision 33415) @@ -47,6 +47,7 @@ } - Xs = image[0].picture.x; - Ys = image[0].picture.y; + // Xs,Ys are in full-frame coords. can we trim here? + Xs = image[0].picture.x - graphic[0].xwin; + Ys = image[0].picture.y - graphic[0].ywin; dx = image[0].picture.dx; dy = image[0].picture.dy; @@ -54,4 +55,5 @@ DY = image[0].image[0].matrix.Naxis[1]; + // the created buffer is supposed to contain the output windows if (buffer[0].Nx < Xs + dx) { fprintf (stderr, "invalid condition\n"); Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawIt.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawIt.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawIt.c (revision 33415) @@ -13,5 +13,9 @@ black = KapaColorByName ("black"); - buffer = bDrawBufferCreate (graphic->dx, graphic->dy, Nbyte, palette, Npalette); + // SectionMinBoundary (graphic); + + // if we want to trim, we'll need to carry about the start in graphic coords and + // the dx,dy size. + buffer = bDrawBufferCreate (graphic->dxwin, graphic->dywin, Nbyte, palette, Npalette); bDrawSetStyle (buffer, black, 0, 0); Index: branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawObjects.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawObjects.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/kapa2/src/bDrawObjects.c (revision 33415) @@ -307,4 +307,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -326,4 +327,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -345,4 +347,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -365,4 +368,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -385,4 +389,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -405,4 +410,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -424,4 +430,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -445,4 +452,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -464,4 +472,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -487,4 +496,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; @@ -511,4 +521,5 @@ { if (scaleColor) { + if (!finite(z[i])) continue; int pixel = MIN (graphic->Npixels - 2, MAX (0, z[i]*(graphic->Npixels - 1))); buffer->bColor_R = pixel1[pixel]; Index: branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dbExtractMeasures.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dbExtractMeasures.c (revision 33415) @@ -245,8 +245,8 @@ break; case MEAS_PAR: /* OK */ - value.Flt = average[0].R; + value.Flt = average[0].P; break; case MEAS_PAR_ERR: /* OK */ - value.Flt = average[0].D; + value.Flt = average[0].dP; break; case MEAS_CHISQ_POS: /* OK */ Index: branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dvo_set_catdir.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dvo_set_catdir.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dvo_set_catdir.c (revision 33415) @@ -15,5 +15,6 @@ } if (catdir) { - current_catdir = strdup(catdir); + // need to keep this in the ohana memory system, so we cannot use strdup + current_catdir = strcreate(catdir); } } Index: branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dvo_util.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dvo_util.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/libdvo/src/dvo_util.c (revision 33415) @@ -276,5 +276,5 @@ FREE(detections); - if (NInvalidDetID) fprintf (stderr, "ERROR: Encountered %ld invalid detection IDs\n", NInvalidDetID); + if (NInvalidDetID) fprintf (stderr, "ERROR: Encountered %zx invalid detection IDs\n", NInvalidDetID); return (Ndetect); Index: branches/meh_branches/ppstack_test/Ohana/src/libkapa/src/PSRotFont.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/libkapa/src/PSRotFont.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/libkapa/src/PSRotFont.c (revision 33415) @@ -32,4 +32,24 @@ dY = currentfont[65].ascent; + /*** + + update this to use the following PS code: + + /ceshow { % (string) fontsize fontname x y + gsave + moveto findfont exch scalefont setfont % s + gsave + dup false charpath flattenpath pathbbox % s x0 y0 x1 y1 + grestore + 3 -1 roll sub % s x0 x1 dy + 3 1 roll sub % s dy -dx + 2 div exch % s -dx/2 dy + -2 div % s -dx/2 -dy/2 + rmoveto show + grestore + } bind def + + ***/ + /* apply appropriate offset */ Xoff = Yoff = 0; Index: branches/meh_branches/ppstack_test/Ohana/src/libohana/include/ohana.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/libohana/include/ohana.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/libohana/include/ohana.h (revision 33415) @@ -20,5 +20,5 @@ // comment this out to avoid the internal Ohana memory management code -// # define OHANA_MEMORY +# define OHANA_MEMORY // XXX I was including these before, but RHL claims they are not needed Index: branches/meh_branches/ppstack_test/Ohana/src/libohana/src/ohana_allocate.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/libohana/src/ohana_allocate.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/libohana/src/ohana_allocate.c (revision 33415) @@ -112,4 +112,7 @@ // memblock of supplied pointer old = (Memblock *) in - 1; + + if (old->startblock != OHANA_MEMMAGIC) ohana_memabort ("corrupt memory (%s, %d, %s)\n", file, line, func); + if (old->endblock != OHANA_MEMMAGIC) ohana_memabort ("corrupt memory (%s, %d, %s)\n", file, line, func); Nelem = MAX (1, Nelem); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/Makefile =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/Makefile (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/Makefile (revision 33415) @@ -43,4 +43,5 @@ $(SRC)/galprofiles.$(ARCH).o \ $(SRC)/elliprofile.$(ARCH).o \ +$(SRC)/ringflux.$(ARCH).o \ $(SRC)/petrosian.$(ARCH).o \ $(SRC)/multifit.$(ARCH).o \ Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/init.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/init.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/init.c (revision 33415) @@ -29,4 +29,5 @@ int galradbins PROTO((int, char **)); int elliprofile PROTO((int, char **)); +int ringflux PROTO((int, char **)); int petrosian PROTO((int, char **)); int mkgauss PROTO((int, char **)); @@ -78,4 +79,5 @@ {1, "galradbins", galradbins, "generate radial vectors with interpolation along paths"}, {1, "elliprofile", elliprofile, "generate radial vectors with interpolation along paths"}, + {1, "ringflux", ringflux, "mean flux in a ring"}, {1, "petrosian", petrosian, "petrosian parameters given radial bins"}, {1, "multifit", multifit, "fit multi-order spectrum"}, Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/ringflux.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/ringflux.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.astro/ringflux.c (revision 33415) @@ -0,0 +1,100 @@ +# include "astro.h" + +# define FLUX(BUF,X,Y) { \ + char valid = TRUE; \ + valid &= (X >= 0); \ + valid &= (Y >= 0); \ + valid &= (X < Nx); \ + valid &= (Y < Ny); \ + if (valid) { \ + out[Nvec] = BUF[(int)(X) + ((int)(Y))*Nx]; \ + Nvec++; \ + if (Nvec >= vec[0].Nelements) abort(); \ + } \ + } + +# define C1 0 + +int ringflux (int argc, char **argv) { + + int Nx, Ny, x, y, d, radius, Radius, dRadius, Nvec; + double *out; + float *in; + float Xo, Yo; + Buffer *buf; + Vector *vec; + + if (argc != 7) { + gprint (GP_ERR, "USAGE: ringflux (buffer) (vector) (Xo) (Yo) (Radius) (dRadius)\n"); + return (FALSE); + } + + /* select input / output buffers */ + if ((buf = SelectBuffer (argv[1], OLDBUFFER, TRUE)) == NULL) return (FALSE); + if ((vec = SelectVector (argv[2], ANYVECTOR, TRUE)) == NULL) return (FALSE); + + Nx = buf[0].header.Naxis[0]; + Ny = buf[0].header.Naxis[1]; + + Xo = atof(argv[3]); + Yo = atof(argv[4]); + + Radius = atoi (argv[5]); + dRadius = atoi (argv[6]); + + // just a rough guess at the number needed + ResetVector (vec, OPIHI_FLT, MAX (6*Radius*dRadius, 8)); + + Nvec = 0; + for (radius = Radius; radius < Radius + dRadius; radius ++) { + + x = 0; + y = radius; + +# if (C1) + d = 3 - 2*radius; +# else + d = 5 - 4*radius; +# endif + + in = (float *) buf[0].matrix.buffer; + out = vec[0].elements.Flt; + + while (x <= y) { + FLUX(in, (Xo + x), (Yo + y)); + FLUX(in, (Xo + x), (Yo - y)); + FLUX(in, (Xo + y), (Yo + x)); + FLUX(in, (Xo - y), (Yo + x)); + + if (x > 0) { + FLUX(in, (Xo - x), (Yo + y)); + FLUX(in, (Xo - x), (Yo - y)); + FLUX(in, (Xo - y), (Yo - x)); + FLUX(in, (Xo + y), (Yo - x)); + } + if (Nvec >= vec[0].Nelements - 8) { + // counting error here + ResetVector (vec, OPIHI_FLT, vec[0].Nelements + 64); + out = vec[0].elements.Flt; + } + + if (d < 0) { +# if (C1) + d = d + 4*x + 6; +# else + d = d + 8*x + 4; +# endif + } else { +# if (C1) + d = d + 4*(x-y) + 10; +# else + d = d + 8*(x-y) + 8; +# endif + y--; + } + x++; + } + } + vec[0].Nelements = Nvec; + return (TRUE); +} Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.basic/echo.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.basic/echo.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.basic/echo.c (revision 33415) @@ -3,5 +3,11 @@ int echo (int argc, char **argv) { - int i; + int i, N, RETURN_CHAR; + + RETURN_CHAR = TRUE; + if ((N = get_argument (argc, argv, "-no-return"))) { + remove_argument (N, &argc, argv); + RETURN_CHAR = FALSE; + } for (i = 1; i < argc - 1; i++) { @@ -9,5 +15,9 @@ } if (argc >= 2) { + if (RETURN_CHAR) { gprint (GP_LOG, "%s\n", argv[argc - 1]); + } else { + gprint (GP_LOG, "%s", argv[argc - 1]); + } } return (TRUE); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/plot.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/plot.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/plot.c (revision 33415) @@ -53,4 +53,9 @@ return (FALSE); } + if (dypvec && (dypvec->Nelements != xvec->Nelements)) goto mismatch; + if (dymvec && (dymvec->Nelements != xvec->Nelements)) goto mismatch; + if (dxpvec && (dxpvec->Nelements != xvec->Nelements)) goto mismatch; + if (dxmvec && (dxmvec->Nelements != xvec->Nelements)) goto mismatch; + Npts = xvec[0].Nelements; if (Npts == 0) return (TRUE); @@ -69,3 +74,7 @@ } return (TRUE); + +mismatch: + gprint (GP_ERR, "error and data vector lengths are mismatched\n"); + return (FALSE); } Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/read_vectors.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/read_vectors.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/read_vectors.c (revision 33415) @@ -23,5 +23,5 @@ int read_vectors (int argc, char **argv) { - int i, j, Nskip, Nvec, *col, done, status, IsCSV; + int i, j, Nskip, Nvec, *col, done, status, IsCSV, VERBOSE; int Nbytes, nbytes, Nstart, NELEM, N, nread; char *colstr, *c0, *c1, *buffer, *extname; @@ -51,4 +51,10 @@ remove_argument (N, &argc, argv); IsCSV = TRUE; + } + + VERBOSE = FALSE; + if ((N = get_argument (argc, argv, "-v"))) { + remove_argument (N, &argc, argv); + VERBOSE = TRUE; } @@ -121,7 +127,4 @@ scan_line (f, buffer); } - - int Nfield = 0; - int Nline = 0; Nstart = 0; @@ -157,17 +160,28 @@ status = dparse (&value, col[i], c0); } - vec[i][0].elements.Flt[N] = value; if (status) { - Nfield ++; + vec[i][0].elements.Flt[N] = value; + } else { + vec[i][0].elements.Flt[N] = NAN; + if (VERBOSE) { + if (IsCSV) { + gprint (GP_ERR, "suspect field: %d (%s) in %s\n", col[i], argv[2*i+2], c0); + } else { + gprint (GP_ERR, "suspect field: %d in %s\n", col[i], c0); + } + } } - if (!status) vec[i][0].elements.Flt[N] = NAN; - } - if (status) N++; - } - Nline ++; - if (c1) { - // fprintf (stderr, "line %d, chars %ld, fields %d\n", Nline, (c1 - c0), Nfield); - } - Nfield = 0; + } + if (status) { + N++; + } else { + if (VERBOSE && FALSE) { + char temp[32]; + strncpy (temp, c0, 32); + temp[31] = 0; + gprint (GP_ERR, "skip line %s\n\n", temp); + } + } + } c0 = c1 + 1; if (N == NELEM) { @@ -177,5 +191,4 @@ } } - } } Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/vstats.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/vstats.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/cmd.data/vstats.c (revision 33415) @@ -169,6 +169,6 @@ // we are going to do another pass: mark the entries to skip - pmin = max; - pmax = min; + pmin = min; + pmax = max; if (iter < Niter - 1) { if (vec[0].type == OPIHI_FLT) { @@ -208,4 +208,5 @@ skip: FREE(mask); + if (mask) mask = NULL; if (!Quiet) { Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/dvo/skycoverage.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/dvo/skycoverage.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/dvo/skycoverage.c (revision 33415) @@ -202,5 +202,5 @@ V[ys*Nx + xs] = ShowDensity ? 0 : 2; } else { - V[ys*Nx + xs] = ShowDensity ? -1 : 0; + V[ys*Nx + xs] = ShowDensity ? NAN : 0; } } Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/include/pantasks.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/include/pantasks.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/include/pantasks.h (revision 33415) @@ -40,4 +40,5 @@ TASK_END, TASK_HOST, + TASK_NICE, TASK_STDOUT, TASK_STDERR, @@ -115,4 +116,5 @@ int active; + int priority; } Task; @@ -159,4 +161,5 @@ JobMode mode; /* local or controller? */ + int priority; char *realhost; Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/include/pcontrol.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/include/pcontrol.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/include/pcontrol.h (revision 33415) @@ -119,4 +119,5 @@ int exit_status; int Reset; + int priority; JobMode mode; JobStat state; @@ -152,4 +153,25 @@ } Host; +# if (USE_LLIST) +typedef struct StackItem { + StackItem *next; + StackItem *prev; + void *object; + char *name; + int id; +} StackItem; + +/* the Jobs and Hosts are managed in a set of Stacks which define their state */ +typedef struct { + StackItem *head; + StackItem *tail; // use this? + int Nobject; +# ifdef THREADED + pthread_mutex_t mutex; +# endif +} Stack; + +# else + /* the Jobs and Hosts are managed in a set of Stacks which define their state */ typedef struct { @@ -163,4 +185,5 @@ # endif } Stack; +# endif /* XXX if this is hard-wired, we can't change shell name in StartHost */ @@ -291,5 +314,5 @@ Job *PullJobByID (IDtype JobID, int *StackID); Job *PullJobFromStackByID (int StackID, int ID); -IDtype AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv, int Nxhosts, char **xhosts); +IDtype AddJob (char *hostname, JobMode mode, int timeout, int priority, int argc, char **argv, int Nxhosts, char **xhosts); void DelJob (Job *job); Host *UnlinkJobAndHost (Job *job); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/include/shell.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/include/shell.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/include/shell.h (revision 33415) @@ -184,7 +184,7 @@ /* socket functions */ -int InitServerSocket (SockAddress *Address); +int InitServerSocket (SockAddress *Address, char *hostname, char *portinfo); int WaitServerSocket (int InitSocket, SockAddress *Address); -int GetClientSocket (char *hostname); +int GetClientSocket (char *hostname, char *portinfo); int InitServerSocket_Named (char *hostname, SockAddress *Address); int DefineValidIP (void); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/lib.shell/SocketOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/lib.shell/SocketOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/lib.shell/SocketOps.c (revision 33415) @@ -1,3 +1,5 @@ # include "shell.h" +# include +# define HOST_NAME_MAX 256 # define MY_PORT 2000 @@ -11,34 +13,25 @@ static int *VALID; -int InitServerSocket (SockAddress *Address) { - +int GetPortRange (int *start, int *stop, char *portinfo); + +int InitServerSocket (SockAddress *Address, char *hostname, char *portinfo) { + + int start, stop; int status, InitSocket, length; - -# if (0) - struct hostent *host; - char tmpline[80], hostip[80]; - - host = gethostbyname (hostname); - bzero (hostip, 80); - for (i = 0; i < host[0].h_length; i++) { - sprintf (tmpline, "%u", (0xff & host[0].h_addr[i])); - strcat (hostip, tmpline); - if (i < host[0].h_length - 1) strcat (hostip, "."); - } -# endif - + char myHostname[HOST_NAME_MAX]; + + status = gethostname (myHostname, HOST_NAME_MAX); + + fprintf (stderr, "target host: %s, real host: %s\n", hostname, myHostname); + + GetPortRange (&start, &stop, portinfo); + + fprintf (stderr, "using port range %d - %d\n", start, stop); + Address[0].sin_family = AF_INET; - Address[0].sin_port = MY_PORT; + Address[0].sin_port = start; Address[0].sin_addr.s_addr = INADDR_ANY; // use this line to bind any address / port? retry_server: - -# if (0) - status = inet_aton (hostip, &Address[0].sin_addr); - if (!status) { - gprint (GP_ERR, "invalid address\n"); - exit (4); - } -# endif length = sizeof(Address[0]); @@ -74,5 +67,5 @@ if (errno == EADDRINUSE) { Address[0].sin_port ++; - if (Address[0].sin_port > MY_PORT + 10) { + if (Address[0].sin_port > stop) { fprintf (stderr, "failed to find a usable port\n"); exit (6); @@ -154,7 +147,49 @@ } -int GetClientSocket (char *hostname) { - - int i, status, InitSocket, length; +int GetPortRange (int *start, int *stop, char *portinfo) { + + // portinfo is a port or port range of the form NN or NN:MM + *start = MY_PORT; + *stop = *start + 10; + if (!portinfo) return TRUE; + if (*portinfo == 0) return TRUE; + + char *endptr; + *start = strtol (portinfo, &endptr, 0); + *stop = *start + 10; // default range of 10 + if (!endptr) { + gprint (GP_ERR, "error in port range parsing : %s\n", portinfo); + exit (20); + } + if (endptr == portinfo) { + gprint (GP_ERR, "error in port range (%s), must be in form NN:MM or NN\n", portinfo); + exit (20); + } + if (*endptr == 0) return TRUE; + + if (*endptr != ':') { + gprint (GP_ERR, "error in port range %s (wrong range separator, must be in form NN:MM)\n", portinfo); + exit (20); + } + char *ptr = endptr + 1; + *stop = strtol (ptr, &endptr, 0); + if (endptr == ptr) { + gprint (GP_ERR, "error in port range (%s), must be in form NN:MM or NN\n", portinfo); + exit (20); + } + if (*stop - *start > 20) { + gprint (GP_ERR, "error in port range (%s), range must be 20 or less\n", portinfo); + exit (20); + } + if (*stop < *start) { + gprint (GP_ERR, "error in port range (%s), stop must >= start\n", portinfo); + exit (20); + } + return TRUE; +} + +int GetClientSocket (char *hostname, char *portinfo) { + + int i, status, InitSocket, length, start, stop; SockAddress Address; struct hostent *host; @@ -169,10 +204,12 @@ } + GetPortRange (&start, &stop, portinfo); + if (DEBUG) { - gprint (GP_ERR, "trying %s (%s:%d)...", host[0].h_name, hostip, MY_PORT); + gprint (GP_ERR, "trying %s (%s:%d)...", host[0].h_name, hostip, start); } Address.sin_family = AF_INET; - Address.sin_port = MY_PORT; + Address.sin_port = start; retry_client: @@ -195,5 +232,5 @@ if (errno == ECONNREFUSED) { Address.sin_port ++; - if (Address.sin_port > MY_PORT + 10) exit (12); + if (Address.sin_port > stop) exit (12); goto retry_client; } Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/lib.shell/multicommand.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/lib.shell/multicommand.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/lib.shell/multicommand.c (revision 33415) @@ -13,5 +13,5 @@ void multicommand_InitServer () { - char hostname[256], PASSWORD[256]; + char hostname[256], PASSWORD[256], portinfo[256]; if (server != 0) { @@ -27,6 +27,10 @@ } + /* is a port range defined? otherwise use the default */ + memset (portinfo, 0, 256); + VarConfig ("PANTASKS_SERVER_PORT", "%s", portinfo); + /* attempt to connect to the server */ - server = GetClientSocket (hostname); + server = GetClientSocket (hostname, portinfo); /* here we can perform the security handshaking */ Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/ControllerOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/ControllerOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/ControllerOps.c (revision 33415) @@ -258,4 +258,10 @@ sprintf (cmd, "job -host %s", job[0].task[0].host); } + } + + if (job[0].priority) { + char tmp[64]; + snprintf (tmp, 64, " -nice %d", job[0].priority); + strcat (cmd, tmp); } Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/JobOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/JobOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/JobOps.c (revision 33415) @@ -107,4 +107,5 @@ job[0].mode = JOB_CONTROLLER; } + job[0].priority = task[0].priority; /* we need our own copy of task[0].argv argc is the number of valid args, like the usual command line. we Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/LocalJob.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/LocalJob.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/LocalJob.c (revision 33415) @@ -161,4 +161,14 @@ if (VerboseMode()) gprint (GP_ERR, "local job launched\n"); + /* set nice level for the child process -- maybe I should not exit here... */ + if (job[0].priority) { + status = setpriority (PRIO_PROCESS, pid, job[0].priority); + if (status == -1) { + gprint (GP_ERR, "error setting priority\n"); + perror ("setpriority: "); + exit (2); + } + } + /* close the other ends of the pipes */ close (stdout_fd[1]); stdout_fd[1] = 0; Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/Makefile =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/Makefile (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/Makefile (revision 33415) @@ -83,4 +83,5 @@ $(SRC)/task.$(ARCH).o \ $(SRC)/task_host.$(ARCH).o \ +$(SRC)/task_nice.$(ARCH).o \ $(SRC)/task_nmax.$(ARCH).o \ $(SRC)/task_active.$(ARCH).o \ Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/TaskOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/TaskOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/TaskOps.c (revision 33415) @@ -487,4 +487,5 @@ NewTask[0].active = TRUE; + NewTask[0].priority = 0; return (NewTask); } @@ -595,4 +596,5 @@ if (!strcasecmp (command, "END")) hash = TASK_END; if (!strcasecmp (command, "HOST")) hash = TASK_HOST; + if (!strcasecmp (command, "NICE")) hash = TASK_NICE; if (!strcasecmp (command, "NMAX")) hash = TASK_NMAX; if (!strcasecmp (command, "ACTIVE")) hash = TASK_ACTIVE; Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/init.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/init.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/init.c (revision 33415) @@ -4,4 +4,5 @@ int task PROTO((int, char **)); int task_host PROTO((int, char **)); +int task_nice PROTO((int, char **)); int task_nmax PROTO((int, char **)); int task_npending PROTO((int, char **)); @@ -33,4 +34,5 @@ {1, "halt", halt, "halt the scheduler (no job harvesting)"}, {1, "host", task_host, "define host machine for a task"}, + {1, "nice", task_nice, "set nice priority level for a task"}, {1, "ipptool2book", ipptool2book, "convert queue with ipptool output to book"}, {1, "kill", kill_job, "kill job"}, Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/init_server.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/init_server.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/init_server.c (revision 33415) @@ -4,4 +4,5 @@ int task PROTO((int, char **)); int task_host PROTO((int, char **)); +int task_nice PROTO((int, char **)); int task_nmax PROTO((int, char **)); int task_npending PROTO((int, char **)); @@ -34,4 +35,5 @@ {1, "delete", delete_job, "delete job"}, {1, "host", task_host, "define host machine for a task"}, + {1, "nice", task_nice, "set nice priority level for a task"}, {1, "ipptool2book", ipptool2book, "convert queue with ipptool output to book"}, {1, "kill", kill_job, "kill job"}, Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/pantasks_server.c.in =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/pantasks_server.c.in (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/pantasks_server.c.in (revision 33415) @@ -18,4 +18,5 @@ int main (int argc, char **argv) { + char hostname[256], portinfo[256]; char log_stdout[1024], log_stderr[1024]; pthread_t JobsAndTasksThread; @@ -100,6 +101,16 @@ pass them to the ListenClient thread */ + /* find the defined server hostname */ + if (VarConfig ("PANTASKS_SERVER", "%s", hostname) == NULL) { + gprint (GP_ERR, "pantasks server host undefined\n"); + exit (31); + } + + /* is a port range defined? otherwise use the default */ + memset (portinfo, 0, 256); + VarConfig ("PANTASKS_SERVER_PORT", "%s", portinfo); + /* create the listening socket */ - InitSocket = InitServerSocket (&Address); + InitSocket = InitServerSocket (&Address, hostname, portinfo); InitPassword (); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/task.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/task.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/task.c (revision 33415) @@ -72,4 +72,5 @@ case TASK_NMAX: case TASK_HOST: + case TASK_NICE: case TASK_EXIT: case TASK_EXEC: Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/task_nice.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/task_nice.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/task_nice.c (revision 33415) @@ -0,0 +1,34 @@ +# include "pantasks.h" + +int task_nice (int argc, char **argv) { + + char *endptr = NULL; + int priority; + Task *task; + + if (argc != 2) { + gprint (GP_ERR, "USAGE: nice (priority)\n"); + return (FALSE); + } + + JobTaskLock(); + task = GetNewTask (); + if (task == NULL) { + gprint (GP_ERR, "ERROR: not defining or running a task\n"); + JobTaskUnlock(); + return (FALSE); + } + + priority = strtol (argv[1], &endptr, 10); + if (*endptr) goto fail; + if (priority < 0) goto fail; + if (priority > 20) goto fail; + + task[0].priority = priority; + JobTaskUnlock(); + return (TRUE); + +fail: + gprint (GP_ERR, "ERROR: nice (priority) -- priority must be an integer 0 to 20\n"); + return (FALSE); +} Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/test/nice.sh =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/test/nice.sh (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/test/nice.sh (revision 33415) @@ -0,0 +1,70 @@ + +# a basic task which just runs 'sleep 10' +task nicetask_local + command sleep 10 + host local + nice 10 + + periods -poll 0.1 + periods -exec 1.0 + periods -timeout 20 + active true + npending 5 + + stdout tmp.txt + stderr tmp.txt + + task.exec + echo "nicetask_local start" + end + + # success + task.exit 0 + echo "nicetask_local stop" + end + + # default exit status + task.exit default + echo "failure: exit status: $EXIT" + end + + # operation times out? + task.exit timeout + echo "timeout" + end +end + +# a basic task which just runs 'sleep 10' +task meantask_local + command sleep 10 + host local + + periods -poll 0.1 + periods -exec 1.0 + periods -timeout 20 + active true + npending 5 + + stdout tmp.txt + stderr tmp.txt + + task.exec + echo "meantask_local start" + end + + # success + task.exit 0 + echo "meantask_local stop" + end + + # default exit status + task.exit default + echo "failure: exit status: $EXIT" + end + + # operation times out? + task.exit timeout + echo "timeout" + end +end + Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/test/nice_remote.sh =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/test/nice_remote.sh (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pantasks/test/nice_remote.sh (revision 33415) @@ -0,0 +1,70 @@ + +# a basic task which just runs 'sleep 10' +task nicetask_local + command sleep 10 + host anyhost + nice 10 + + periods -poll 0.1 + periods -exec 1.0 + periods -timeout 20 + active true + npending 5 + + stdout tmp.txt + stderr tmp.txt + + task.exec + echo "nicetask_local start" + end + + # success + task.exit 0 + echo "nicetask_local stop" + end + + # default exit status + task.exit default + echo "failure: exit status: $EXIT" + end + + # operation times out? + task.exit timeout + echo "timeout" + end +end + +# a basic task which just runs 'sleep 10' +task meantask_local + command sleep 10 + host anyhost + + periods -poll 0.1 + periods -exec 1.0 + periods -timeout 20 + active true + npending 5 + + stdout tmp.txt + stderr tmp.txt + + task.exec + echo "meantask_local start" + end + + # success + task.exit 0 + echo "meantask_local stop" + end + + # default exit status + task.exit default + echo "failure: exit status: $EXIT" + end + + # operation times out? + task.exit timeout + echo "timeout" + end +end + Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pclient/job.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pclient/job.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pclient/job.c (revision 33415) @@ -3,6 +3,13 @@ int job (int argc, char **argv) { - int i, pid, status; + int i, N, pid, status, priority; char **targv; + + priority = 0; + if ((N = get_argument (argc, argv, "-nice"))) { + remove_argument (N, &argc, argv); + priority = atoi (argv[N]); + remove_argument (N, &argc, argv); + } if (argc < 2) { @@ -54,4 +61,14 @@ } + /* set nice level for the child process */ + if (priority) { + status = setpriority (PRIO_PROCESS, pid, priority); + if (status == -1) { + gprint (GP_ERR, "error setting priority\n"); + perror ("setpriority: "); + exit (2); + } + } + /* free temporary arg list */ for (i = 0; i < argc - 1; i++) { Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/CheckIdleHost.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/CheckIdleHost.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/CheckIdleHost.c (revision 33415) @@ -5,4 +5,27 @@ // a (temporary?) work-around for the problem that the remote pclient job tends to grow // too large over time. + +/**** + + queueing strategy: + + We have a problem when the queue contains many jobs of different processing times. If we + start with an equal number of jobs in two classes, fast and slow, eventually, we will end + up with all hosts running the slow jobs and the fast jobs completely blocked. If J1 takes + T1 and J2 takes T2, and jobs have equal probability to land in a slot... ?? + + It seems like we should boost the probability of the fast jobs over the slow jobs (of + course, the end result of that will be all fast jobs draining and the slow jobs hanging + around). + + There is also a problem related to LAP, in that chip stage has a huge number of tasks in + the queue (effectively infinite). Make the probability of the job being selected + proportional to something. In any case, we need a user-controllable way to change the + probability of selection + + As things are implemented below, the selection order depends on the queue order. The + easiest way to modify the probabilities is to re-sort based on something. + + ****/ static float MAX_WANTHOST_WAIT = 10.0; Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/JobOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/JobOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/JobOps.c (revision 33415) @@ -195,5 +195,5 @@ } -IDtype AddJob (char *hostname, JobMode mode, int timeout, int argc, char **argv, int Nxhosts, char **xhosts) { +IDtype AddJob (char *hostname, JobMode mode, int timeout, int priority, int argc, char **argv, int Nxhosts, char **xhosts) { int JobID; @@ -220,4 +220,5 @@ job[0].mode = mode; + job[0].priority = priority; job[0].state = 0; @@ -270,2 +271,35 @@ FREE (job); } + +/*** ResortJobStack can be used to adjust priorities based on some info we supply. This + is not finished -- to finish this, I need to define the metric of interest and + arrange for that to be passed to the jobs in pcontrol + +void ResortJobStack (int StackID) { + + Stack *stack = GetJobStack (StackID); + LockStack (stack); + +# define SWAPFUNC(A,B){ \ + void *tmpObject = stack[0].object[A]; \ + stack[0].object[A] = stack[0].object[B]; \ + stack[0].object[B] = tmpObject; \ + void *tmpName = stack[0].name[A]; \ + stack[0].name[A] = stack[0].name[B]; \ + stack[0].name[B] = tmpName; \ + void *tmpID = stack[0].id[A]; \ + stack[0].id[A] = stack[0].id[B]; \ + stack[0].id[B] = tmpID; \ + } + +# define COMPARE(A,B)(((Job *)stack[0].object[A]).VALUE < ((Job *)stack[0].object[B]).VALUE) + + OHANA_SORT (stack[0].Nobject, COMPARE, SWAPFUNC); + +# undef SWAPFUNC +# undef COMPARE + + UnlockStack (stack); +} + +***/ Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/StackOpsLL.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/StackOpsLL.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/StackOpsLL.c (revision 33415) @@ -0,0 +1,227 @@ +# include "pcontrol.h" + +/* Stacks and thread locks: interacting with the Stacks needs to be thread-safe so that the user may + * perform operations which interact with the stacks at the same time that the background loops + * check the current status of the jobs and hosts in the different stacks. The simplest way in + * which the stacks are made thread safe is to lock them with a mutex before every interaction + */ + +# define DEBUG 0 + +void PrintStackInfo (Stack *stack, const char *func) { + + if (!DEBUG) return; + fprintf (stderr, "%s: %p ", func, stack); + fprintf (stderr, "objects: %p ", stack[0].object); + fprintf (stderr, "Nobjects: %d (linked list)\n", stack[0].Nobject); +} + +/* allocate stack: create head node (empty node?) */ +Stack *InitStack () { + + Stack *stack; + + ALLOCATE (stack, Stack, 1); + + stack->Nobject = 0; + stack->head = NULL; + stack->tail = NULL; + +# ifdef THREADED + pthread_mutex_init (&stack[0].mutex, NULL); +# endif + return (stack); +} + +void FreeStack (Stack *stack) { + + ASSERT (!stack->head, "stack items not freed"); + ASSERT (!stack->tail, "stack items not freed"); + free (stack); +} + +/* STACK_TOP == 0, STACK_BOTTOM == -1 + in this version, entries by seq number other than head and tail are not allowed + */ + +StackItem InitStackItem (void *object, char *name, int id) { + + StackItem *item; + ALLOCATE (item, StackItem, 1); + item->object = object; + item->name = name; + item->id = id; + + return item; +} + +/* push object on stack at given location */ +int PushStack (Stack *stack, int where, void *object, int id, char *name) { + + int i; + + ASSERT (where == STACK_TOP || where == STACK_BOTTOM, "invalid location"); + ASSERT (stack != NULL, "stack not set"); + + PrintStackInfo (stack, __func__); + LockStack (stack); + + // make this a StackItem generator + StackItem *item = InitStackItem (object, name, id); + ALLOCATE (item, StackItem, 1); + + /* there is a special case when the stack is empty. in this case, we have to update + both head and tail. + */ + if (!stack->head && !stack->tail) { + stack->head = item; + stack->tail = item; + stack->Nobject ++; + UnlockStack (stack); + return (TRUE); + } + + if (where == STACK_TOP) { + StackHead *oldHead = stack->head; + stack->head = item; + item->next = oldHead; + oldHead->prev = item; + } else { + StackHead *oldTail = stack->tail; + stack->tail = item; + item->prev = oldTail; + oldTail->next = item; + } + stack->Nobject ++; + UnlockStack (stack); + return (TRUE); +} + +/* get object from specified point in stack (negative == distance from end) */ +void *PullStackByLocation (Stack *stack, int where) { + + void *object; + + ASSERT (where == STACK_TOP || where == STACK_BOTTOM, "invalid location"); + ASSERT (stack != NULL, "stack not set"); + + PrintStackInfo (stack, __func__); + LockStack (stack); + + if ( + + object = stack[0].object[where]; + RemoveStackEntry (stack, where); + UnlockStack (stack); + return (object); +} + +/* get object from stack which matches name */ +void *PullStackByName (Stack *stack, char *name) { + + int i; + void *object; + + PrintStackInfo (stack, __func__); + ASSERT (stack != NULL, "stack not set"); + LockStack (stack); + + for (i = 0; i < stack[0].Nobject; i++) { + if (strcasecmp (stack[0].name[i], name)) continue; + + /* here is the element of interest */ + object = stack[0].object[i]; + RemoveStackEntry (stack, i); + UnlockStack (stack); + return (object); + } + UnlockStack (stack); + return (NULL); +} + +/* get object from point in stack (negative == distance from end) */ +void *PullStackByID (Stack *stack, int id) { + + int i; + void *object; + + PrintStackInfo (stack, __func__); + ASSERT (stack != NULL, "stack not set"); + LockStack (stack); + + for (i = 0; i < stack[0].Nobject; i++) { + if (stack[0].id[i] != id) continue; + + /* here is the element of interest */ + object = stack[0].object[i]; + RemoveStackEntry (stack, i); + UnlockStack (stack); + return (object); + } + UnlockStack (stack); + return (NULL); +} + +/* should only be called if you know where is a valid entry */ +int RemoveStackEntry (Stack *stack, int where) { + + int i; + + PrintStackInfo (stack, __func__); + ASSERT (stack != NULL, "stack not set"); + + if (where < 0) abort(); + if (where >= stack[0].Nobject) abort(); + if (stack[0].Nobject < 1) abort(); + + /* shift the remaining entries by one */ + /* XXX free associated memory */ + stack[0].Nobject --; + for (i = where; i < stack[0].Nobject; i++) { + stack[0].object[i] = stack[0].object[i+1]; + stack[0].name[i] = stack[0].name[i+1]; + stack[0].id[i] = stack[0].id[i+1]; + } + return (TRUE); +} + +/* should only be called if you manually lock the stack */ +void *RemoveStackByID (Stack *stack, int id) { + + int i; + void *object; + + PrintStackInfo (stack, __func__); + ASSERT (stack != NULL, "stack not set"); + for (i = 0; i < stack[0].Nobject; i++) { + if (stack[0].id[i] != id) continue; + + /* here is the element of interest */ + object = stack[0].object[i]; + RemoveStackEntry (stack, i); + return (object); + } + return (NULL); +} + +void LockStack (Stack *stack) { +# ifdef THREADED + PrintStackInfo (stack, __func__); + ASSERT (stack != NULL, "stack not set"); + pthread_mutex_lock (&stack[0].mutex); +# endif + return; +} + +void UnlockStack (Stack *stack) { +# ifdef THREADED + PrintStackInfo (stack, __func__); + ASSERT (stack != NULL, "stack not set"); + pthread_mutex_unlock (&stack[0].mutex); +# endif + return; +} + +/*** I need a function to sort a stack by something. + I also need to replace these arrays with linked lists. + ***/ Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/StartJob.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/StartJob.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/StartJob.c (revision 33415) @@ -25,4 +25,9 @@ bzero (line, Nline); strcpy (line, "job"); + if (job[0].priority) { + char tmp[64]; + snprintf (tmp, 64, " -nice %d", job[0].priority); + strcat (line, tmp); + } for (i = 0; i < job[0].argc; i++) { strcat (line, " "); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/job.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/job.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/job.c (revision 33415) @@ -5,5 +5,5 @@ char *Host = NULL; char **targv = NULL; - int i, N, Mode, targc, Timeout; + int i, N, Mode, targc, Timeout, priority; IDtype JobID; char **xhosts = NULL; @@ -43,4 +43,11 @@ } + priority = 0; + if ((N = get_argument (argc, argv, "-nice"))) { + remove_argument (N, &argc, argv); + priority = atoi (argv[N]); + remove_argument (N, &argc, argv); + } + xhosts = NULL; Nxhosts = 0; @@ -60,8 +67,5 @@ } - if (argc < 2) { - FREE (Host); - goto usage; - } + if (argc < 2) goto usage; targc = argc - 1; @@ -72,5 +76,5 @@ // a JobID < 0 mean the job was not accepted - JobID = AddJob (Host, Mode, Timeout, targc, targv, Nxhosts, xhosts); + JobID = AddJob (Host, Mode, Timeout, priority, targc, targv, Nxhosts, xhosts); gprint (GP_LOG, "JobID: %d\n", (int) JobID); return (TRUE); Index: branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/status.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/status.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/opihi/pcontrol/status.c (revision 33415) @@ -59,8 +59,9 @@ } + PrintID (GP_LOG, job[0].JobID); + gprint (GP_LOG, " %2d ", job[0].priority); for (j = 0; j < job[0].argc; j++) { gprint (GP_LOG, "%s ", job[0].argv[j]); } - PrintID (GP_LOG, job[0].JobID); gprint (GP_LOG, "\n"); } Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/Makefile =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/Makefile (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/Makefile (revision 33415) @@ -62,4 +62,6 @@ $(SRC)/high_speed_objects.$(ARCH).o \ $(SRC)/high_speed_utils.$(ARCH).o \ +$(SRC)/relastro_merge_source.$(ARCH).o \ +$(SRC)/resort_catalog.$(ARCH).o \ $(SRC)/relastroVisual.$(ARCH).o Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/doc/2M1835+32_xxxx_astrom_summary.txt =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/doc/2M1835+32_xxxx_astrom_summary.txt (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/doc/2M1835+32_xxxx_astrom_summary.txt (revision 33415) @@ -0,0 +1,9 @@ +# JD dT (yr) RA (deg) Dec (deg) dRA_nopm dDe_nopm dRA_O-C dDe_O-C dRA_err dDe_err Nfr FWHM S/N parms +2455289.46232 11.952044 278.907656228 32.996014203 212.415 62.135 11.086 45.005 16.240 15.494 1 0.000 -NaN -0.068662 00 0.7 2.9 94.704 +2450923.97830 0.000000 278.907921440 32.998500058 188.709 67.052 -0.110 8.130 70.000 70.000 1 0.000 -NaN -0.748337 01 -0.0 0.1 1.537 +2455449.75502 12.390901 278.907522057 32.995890174 -234.674 -55.956 -36.700 -101.438 21.245 21.245 1 0.000 -NaN 0.173250 02 -1.7 -4.8 1.032 +2455449.75984 12.390915 278.907530212 32.995944993 -205.316 141.404 -7.336 95.933 21.164 21.164 1 0.000 -NaN 0.000000 03 -0.3 4.5 1.036 +2455459.75031 12.418267 278.907538552 32.995895676 -173.054 -15.668 34.093 -37.165 21.219 21.219 1 0.000 -NaN 0.000000 04 1.6 -1.8 1.055 +2455459.76131 12.418297 278.907532082 32.995902888 -196.341 10.315 10.812 -11.155 21.191 21.191 1 0.000 -NaN 0.000000 05 0.5 -0.5 1.073 +2455668.07657 12.988633 278.907629935 32.995793586 202.620 43.635 9.726 -6.002 21.200 21.200 1 0.000 -NaN 0.000000 06 0.5 -0.3 1.095 +2455668.08611 12.988659 278.907621238 32.995797113 171.312 56.350 -21.572 6.692 21.256 21.256 1 0.000 -NaN 0.000000 07 -1.0 0.3 1.075 Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/include/relastro.h =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/include/relastro.h (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/include/relastro.h (revision 33415) @@ -15,5 +15,5 @@ typedef enum {FIT_NONE, FIT_AVERAGE, FIT_PM_ONLY, FIT_PAR_ONLY, FIT_PM_AND_PAR} FitMode; -typedef enum {TARGET_NONE, TARGET_OBJECTS, TARGET_SIMPLE, TARGET_CHIPS, TARGET_MOSAICS, TARGET_HIGH_SPEED} FitTarget; +typedef enum {TARGET_NONE, TARGET_OBJECTS, TARGET_SIMPLE, TARGET_CHIPS, TARGET_MOSAICS, TARGET_HIGH_SPEED, TARGET_MERGE_SOURCE} FitTarget; typedef struct { @@ -94,4 +94,9 @@ char SKY_TABLE[256]; int SKY_DEPTH; /** XXX EAM : depth of catalog tables, fix usage */ + +unsigned int OBJ_ID_SRC; +unsigned int CAT_ID_SRC; +unsigned int OBJ_ID_DST; +unsigned int CAT_ID_DST; double SIGMA_LIM; @@ -210,10 +215,10 @@ void getfullregion PROTO((Image *image, off_t Nimage, GSCRegion *fullregion)); Image *getimage PROTO((off_t N)); -Image *getimages PROTO((off_t *N)); +Image *getimages PROTO((off_t *N, off_t **line_number)); void global_stats PROTO((Catalog *catalog, int Ncatalog)); void initGrid PROTO((int dX, int dY)); void initGridBins PROTO((Catalog *catalog, int Ncatalog)); void initImageBins PROTO((Catalog *catalog, int Ncatalog, int FULLINIT)); -void initImages PROTO((Image *input, off_t N)); +void initImages PROTO((Image *input, off_t *line_number, off_t N)); void initMosaicBins PROTO((Catalog *catalog, int Ncatalog)); void initMosaicGrid PROTO((Image *image, off_t Nimage)); @@ -310,9 +315,9 @@ int MeasFilterTest(Measure *measure, int applySigmaLim); -int sun_ecliptic (double jd, double *lambda, double *beta, double *epsilon); -int ParFactor (double *pR, double *pD, double R, double D, double T, double Tmean); -int FitPM (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, int Npts); +int sun_ecliptic (double jd, double *lambda, double *beta, double *epsilon, double *Radius); +int ParFactor (double *pR, double *pD, double RA, double Dec, double Time, double Tmean); +int FitPM (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, int Npts, int XVERB); int FitPar (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *pR, double *pD, int Npts); -int FitPMandPar (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, double *pR, double *pD, int Npts); +int FitPMandPar (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, double *pR, double *pD, int Npts, int XVERB); Mosaic *getMosaicForImage (off_t N); @@ -372,2 +377,5 @@ int applyConstraintsB(Catalog *catalog, off_t i); void setupAreaSelection(SkyRegion *region); + +int relastro_merge_source (); +void resort_catalog (Catalog *catalog); Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/CoordOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/CoordOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/CoordOps.c (revision 33415) @@ -13,5 +13,5 @@ Image *images; - images = getimages (&N); + images = getimages (&N, NULL); NoldCoords = N; Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FitPM.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FitPM.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FitPM.c (revision 33415) @@ -2,5 +2,5 @@ /* do we want an init function which does the alloc and a clear function to free? */ -int FitPM (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, int Npts) { +int FitPM (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, int Npts, int XVERB) { int i; @@ -73,8 +73,9 @@ chisq = 0.0; for (i = 0; i < Npts; i++) { - Xf = fit[0].Ro + fit[0].uR*T[i]; - Yf = fit[0].Do + fit[0].uD*T[i]; - chisq += SQ(X[i] - Xf) / SQ(dX[i]); - chisq += SQ(Y[i] - Yf) / SQ(dY[i]); + Xf = fit[0].Ro + fit[0].uR*T[i]; + Yf = fit[0].Do + fit[0].uD*T[i]; + chisq += SQ(X[i] - Xf) / SQ(dX[i]); + chisq += SQ(Y[i] - Yf) / SQ(dY[i]); + if (XVERB) fprintf (stderr, "chisq contrib : %f %f : %f %f : %f %f : %f %f : %f\n", Xf, Yf, X[i] - Xf, Y[i] - Yf, dX[i], dY[i], (X[i] - Xf) / dX[i], (Y[i] - Yf) / dY[i], chisq); } fit[0].Nfit = Npts; Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FitPMandPar.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FitPMandPar.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FitPMandPar.c (revision 33415) @@ -2,5 +2,5 @@ /* do we want an init function which does the alloc and a clear function to free? */ -int FitPMandPar (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, double *pR, double *pD, int Npts) { +int FitPMandPar (PMFit *fit, double *X, double *dX, double *Y, double *dY, double *T, double *pR, double *pD, int Npts, int XVERB) { int i; @@ -9,4 +9,5 @@ double wx, wy, Wx, Wy, Tx, Ty, Tx2, Ty2, Xs, Ys, XT, YT; double PR, PD, PRT, PDT, PRX, PDY, PR2, PD2; + double chisq, Xf, Yf; A = array_init (5, 5); @@ -95,4 +96,18 @@ /* get the chisq from the matrix values */ + // add up the chi square for the fit + chisq = 0.0; + for (i = 0; i < Npts; i++) { + Xf = fit[0].Ro + fit[0].uR*T[i] + fit[0].p*pR[i]; + Yf = fit[0].Do + fit[0].uD*T[i] + fit[0].p*pD[i]; + chisq += SQ(X[i] - Xf) / SQ(dX[i]); + chisq += SQ(Y[i] - Yf) / SQ(dY[i]); + if (XVERB) fprintf (stderr, "chisq contrib : %f %f : %f %f : %f %f : %f %f : %f\n", Xf, Yf, X[i] - Xf, Y[i] - Yf, dX[i], dY[i], (X[i] - Xf) / dX[i], (Y[i] - Yf) / dY[i], chisq); + + } + fit[0].Nfit = Npts; + + // the reduced chisq is divided by (Ndof = 2*Npts - 5) + fit[0].chisq = chisq / (2.0*Npts - 5.0); return (TRUE); } Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FixProblemImages.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FixProblemImages.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/FixProblemImages.c (revision 33415) @@ -15,5 +15,5 @@ ALLOCATE (sublist.filename, char *, 1); - image = getimages (&Nimage); + image = getimages (&Nimage, NULL); Nbad = 0; Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/ImageOps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/ImageOps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/ImageOps.c (revision 33415) @@ -11,4 +11,5 @@ static Image *image; // list of available images static off_t Nimage; // number of available images +static off_t *LineNumber; // match of subset to full image table static int *Ncatlist; // catalogs associated with each image @@ -27,6 +28,8 @@ # endif -Image *getimages (off_t *N) { +Image *getimages (off_t *N, off_t **line_number) { + *N = Nimage; + if (line_number) *line_number = LineNumber; return (image); } @@ -42,9 +45,10 @@ } -void initImages (Image *input, off_t N) { +void initImages (Image *input, off_t *line_number, off_t N) { off_t i; image = input; + LineNumber = line_number; Nimage = N; @@ -909,4 +913,5 @@ if (!finite(measure[0].dR) || !finite(measure[0].dD)) return FALSE; if (!finite(measure[0].M)) return FALSE; //XXX is this necessary for all relastro tasks? + if (!finite(measure[0].dM)) return FALSE; //XXX is this necessary for all relastro tasks? /* select measurements by photcode, or equiv photcode, if specified */ Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/ParFactor.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/ParFactor.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/ParFactor.c (revision 33415) @@ -24,4 +24,5 @@ # endif +# if (0) /* code borrowed from Skycalc : fix this stuff XXX */ /* Low precision formulae for the sun, from Almanac p. C24 (1990) */ @@ -40,23 +41,57 @@ return TRUE; } +# endif + +/* Low precision formulae for the sun, from Astro. Almanac p. C5 (2012) */ +int sun_ecliptic (double jd, double *lambda, double *beta, double *epsilon, double *Radius) { + +# define J2000 2451545. /* Julian date at standard epoch */ + + double n = jd - J2000; // day number + double L = 280.460 + 0.9856474 * n; // mean solar longitute (corr. for aberration) + double g = (357.528 + 0.9856003 * n)*RAD_DEG; // Mean anomaly + + *lambda = L + 1.915 * sin(g) + 0.020 * sin(2*g); // solar longitude in degrees + *beta = 0.0; // approx latitude + *epsilon = (23.439 - 0.0000004 * n); // obliquity of ecliptic in degrees + *Radius = 1.00014 - 0.01671*cos(g) - 0.00014*cos(2*g); // earth-to-sun dist in AU + return TRUE; +} /* given RA, DEC, Time, calculate the parallax factor */ -int ParFactor (double *pR, double *pD, double R, double D, double T, double Tmean) { +int ParFactor (double *pR, double *pD, double RA, double DEC, double Time, double Tmean) { - double jd; - double L, B, E, e, s, r, d; + double jd, lambda, beta, epsilon, Radius; /* given a time T in UNIX seconds, determine the solar longitude S */ - jd = ohana_sec_to_jd (365.25*86400.0*(T + Tmean)); - sun_ecliptic (jd, &L, &B, &E); + jd = ohana_sec_to_jd (365.25*86400.0*(Time + Tmean)); + sun_ecliptic (jd, &lambda, &beta, &epsilon, &Radius); - e = E * RAD_DEG; - s = L * RAD_DEG; - r = R * RAD_DEG; - d = D * RAD_DEG; - - *pR = +(cos(e)*sin(s)*cos(r) - cos(s)*sin(r)); - *pD = -(cos(e)*sin(s)*sin(r) + cos(s)*cos(r))*sin(d) + sin(e)*sin(s)*cos(d); + double lambda_rad = lambda*RAD_DEG; + double epsilon_rad = epsilon*RAD_DEG; + double RA_rad = RA*RAD_DEG; + double DEC_rad = DEC*RAD_DEG; + + double x = Radius*cos(lambda_rad); + double y = Radius*cos(epsilon_rad)*sin(lambda_rad); + double z = Radius*sin(epsilon_rad)*sin(lambda_rad); + + // original terms: + // *pR = +(cos(e)*sin(s)*cos(r) - cos(s)*sin(r)); + // *pD = -(cos(e)*sin(s)*sin(r) + cos(s)*cos(r))*sin(d) + sin(e)*sin(s)*cos(d); + + // convert e->eps, s->lam, etc + // *pR = +(cos(eps)*sin(lam)*cos(ra) - cos(lam)*sin(ra)); + // *pD = -(cos(eps)*sin(lam)*sin(ra) + cos(lam)*cos(ra))*sin(dec) + sin(eps)*sin(lam)*cos(dec); + + // convert to x,y,z + // *pR = +(y*cos(ra) - x*sin(ra)); + // *pD = -(y*sin(ra) + x*cos(ra))*sin(dec) + z*cos(dec); + + // NOTE: seems to be identical to the old values, except I now include the varying solar distance + *pR = +(y*cos(RA_rad) - x*sin(RA_rad)); + *pD = -(y*sin(RA_rad) + x*cos(RA_rad))*sin(DEC_rad) + z*cos(DEC_rad); + return TRUE; } Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/StarMaps.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/StarMaps.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/StarMaps.c (revision 33415) @@ -40,5 +40,5 @@ off_t i, Nimages; - images = getimages(&Nimages); + images = getimages(&Nimages, NULL); ALLOCATE (starmap, StarMap, Nimages); @@ -64,5 +64,5 @@ gettimeofday (&start, (void *) NULL); - images = getimages(&Nimages); + images = getimages(&Nimages, NULL); for (i = 0; i < catalog[0].Nmeasure; i++) { @@ -90,5 +90,5 @@ int ix, iy; - images = getimages(&Nimages); + images = getimages(&Nimages, NULL); for (i = 0; i < Nimages; i++) { @@ -129,5 +129,5 @@ double L, M, dLmax, dMmax; - images = getimages(&Nimages); + images = getimages(&Nimages, NULL); dLmax = dMmax = 0.0; Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateChips.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateChips.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateChips.c (revision 33415) @@ -18,5 +18,5 @@ Nskip = Nmosaic = NnewFit = NoldFit = 0; - image = getimages (&Nimage); + image = getimages (&Nimage, NULL); // save fit results for summary plot Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateMeasures.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateMeasures.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateMeasures.c (revision 33415) @@ -13,5 +13,5 @@ ID_IMAGE_ASTROM_FEW; - image = getimages (&Nimage); + image = getimages (&Nimage, NULL); for (i = 0; i < Nimage; i++) { Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateObjects.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateObjects.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateObjects.c (revision 33415) @@ -84,4 +84,6 @@ /* calculate the average value of R,D for a single star */ + XVERB = FALSE; + // skip objects which are known to be problematic // XXX include this code or not? @@ -196,5 +198,8 @@ } - XVERB = FALSE && (catalog[i].measure[m].dM < 0.01) && (N == 6) && (mode == FIT_PM_ONLY); + // XVERB |= (catalog[i].averge[j].objID == 0xc90) && (catalog[i].average[j].catID == 0x2a1e); + XVERB |= (catalog[i].average[j].objID == OBJ_ID_SRC) && (catalog[i].average[j].catID == CAT_ID_SRC); + XVERB |= (catalog[i].average[j].objID == OBJ_ID_DST) && (catalog[i].average[j].catID == CAT_ID_DST); + // XVERB = (catalog[i].measure[m].dM < 0.01) && (N == 6) && (mode == FIT_PM_ONLY); // to judge the quality of the PM and PAR fits, we need to fit all three models and compare Chisq @@ -211,7 +216,7 @@ } - FitPM (&fitPM, X, dX, Y, dY, T, N); - - if (XVERB) fprintf (stderr, "fitted: %f - %f : %f %f : %f %f : %f vs %f\n", Tmin, Tmax, fitPM.Ro, fitPM.Do, fitPM.uR, fitPM.uD, fitPM.chisq, fitAve.chisq); + FitPM (&fitPM, X, dX, Y, dY, T, N, XVERB); + + if (XVERB) fprintf (stderr, "fitted PM: %f - %f : %f %f : %f %f : %f vs %f\n", Tmin, Tmax, fitPM.Ro, fitPM.Do, fitPM.uR, fitPM.uD, fitPM.chisq, fitAve.chisq); // project Ro, Do back to RA,DEC @@ -225,11 +230,10 @@ if (mode == FIT_PM_AND_PAR) { - fprintf (stderr, "parallax fitting is still untested (%s, %d)", __FILE__, __LINE__); - exit (2); + // fprintf (stderr, "parallax fitting is still untested (%s, %d)\n", __FILE__, __LINE__); for (k = 0; k < N; k++) { ParFactor (&pX[k], &pY[k], R[k], D[k], T[k], Tmean); } - FitPMandPar (&fitPAR, X, dX, Y, dY, T, pX, pY, N); + FitPMandPar (&fitPAR, X, dX, Y, dY, T, pX, pY, N, XVERB); XY_to_RD (&fitPAR.Ro, &fitPAR.Do, fitPAR.Ro, fitPAR.Do, &coords); catalog[i].average[j].flags |= ID_STAR_FIT_PAR; @@ -276,5 +280,5 @@ break; } - if (XVERB) fprintf (stderr, "A%f %f -> %f %f (%f,%f) pm=(%f %f)\n", + if (XVERB) fprintf (stderr, "%f %f -> %f %f (%f,%f) pm=(%f %f)\n", catalog[i].average[j].R, catalog[i].average[j].D, Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateSimple.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateSimple.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/UpdateSimple.c (revision 33415) @@ -8,5 +8,5 @@ StarData *raw, *ref; - image = getimages (&Nimage); + image = getimages (&Nimage, NULL); for (i = 0; i < Nimage; i++) { Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/args.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/args.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/args.c (revision 33415) @@ -1,4 +1,6 @@ # include "relastro.h" void usage (void); +void usage_merge_source (void); +void usage_merge_source_id (char *name); int args (int argc, char **argv) { @@ -6,11 +8,61 @@ int N; double trange; + char *endptr; /* possible operations */ FIT_TARGET = TARGET_NONE; FIT_MODE = FIT_AVERAGE; + + OBJ_ID_SRC = OBJ_ID_DST = 0; + CAT_ID_SRC = CAT_ID_DST = 0; + + if ((N = get_argument (argc, argv, "-merge-source"))) { + if (N > argc - 6) usage_merge_source(); + if (strcmp(argv[N+3], "into")) usage_merge_source(); + FIT_TARGET = TARGET_MERGE_SOURCE; + remove_argument (N, &argc, argv); + OBJ_ID_SRC = strtol(argv[N], &endptr, 0); + if (*endptr) usage_merge_source_id (argv[N]); + remove_argument (N, &argc, argv); + CAT_ID_SRC = strtol(argv[N], &endptr, 0); + if (*endptr) usage_merge_source_id (argv[N]); + remove_argument (N, &argc, argv); + remove_argument (N, &argc, argv); // remove the 'into' + OBJ_ID_DST = strtol(argv[N], &endptr, 0); + if (*endptr) usage_merge_source_id (argv[N]); + remove_argument (N, &argc, argv); + CAT_ID_DST = strtol(argv[N], &endptr, 0); + if (*endptr) usage_merge_source_id (argv[N]); + remove_argument (N, &argc, argv); + + if (argc != 1) usage (); + return TRUE; + } + if ((N = get_argument (argc, argv, "-update-objects"))) { remove_argument (N, &argc, argv); FIT_TARGET = TARGET_OBJECTS; + } + + if ((N = get_argument (argc, argv, "-testobj1"))) { + if (N > argc - 3) usage (); + remove_argument (N, &argc, argv); + OBJ_ID_SRC = strtol(argv[N], &endptr, 0); + if (*endptr) usage (); + remove_argument (N, &argc, argv); + CAT_ID_SRC = strtol(argv[N], &endptr, 0); + if (*endptr) usage (); + remove_argument (N, &argc, argv); + } + + if ((N = get_argument (argc, argv, "-testobj2"))) { + if (N > argc - 3) usage (); + remove_argument (N, &argc, argv); + OBJ_ID_DST = strtol(argv[N], &endptr, 0); + if (*endptr) usage (); + remove_argument (N, &argc, argv); + CAT_ID_DST = strtol(argv[N], &endptr, 0); + if (*endptr) usage (); + remove_argument (N, &argc, argv); } @@ -307,4 +359,6 @@ fprintf (stderr, "ERROR: USAGE: relastro -region RA RA DEC DEC\n"); fprintf (stderr, " OR: relastro -catalog (ra) (dec)\n\n"); + fprintf (stderr, " OR: relastro -high-speed [options]\n\n"); + fprintf (stderr, " OR: relastro -merge-source [options]\n\n"); fprintf (stderr, " specify one of the following modes: \n"); fprintf (stderr, " -update-objects\n"); @@ -340,2 +394,12 @@ } +void usage_merge_source_id (char *name) { + + fprintf (stderr, "ERROR: invalid ID %s (remember to prefix 0x to hex IDs)\n", name); + exit (2); +} + +void usage_merge_source () { + fprintf (stderr, "ERROR: USAGE: relastro -merge-source (objID) (catID) into (objID) (catID)\n"); + exit (2); +} Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/initialize.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/initialize.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/initialize.c (revision 33415) @@ -10,4 +10,6 @@ ConfigInit (&argc, argv); args (argc, argv); + + if (FIT_TARGET == TARGET_MERGE_SOURCE) return; /* build a list of accepted photcodes. these will be used by bcatalog to accept or Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/load_images.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/load_images.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/load_images.c (revision 33415) @@ -44,14 +44,5 @@ MARKTIME(" select images: %f sec\n", dtime); - // generate db->vtable from db->ftable based on the selection - // XXX does this simply duplicate the memory needlessly? we recreate these lines - // in reload_images. If we had saved the line numbers, we could avoid this - // vtable points *another* copy of the subset rows - // (the later call to 'reload_images' copies the subset elements back on top of - // the rows of the vtable) - // gfits_vtable_from_ftable (&db[0].ftable, &db[0].vtable, LineNumber, Nsubset); - // MARKTIME("converted ftable to vtable: %f sec\n", dtime); - - initImages (subset, Nsubset); + initImages (subset, LineNumber, Nsubset); MARKTIME(" init images: %f sec\n", dtime); @@ -68,9 +59,10 @@ Image *image; - off_t Nimage, Nx, i; + off_t Nimage, Nx, i, *LineNumber; VTable *vtable; - image = getimages (&Nimage); + image = getimages (&Nimage, &LineNumber); + gfits_vtable_from_ftable (&db[0].ftable, &db[0].vtable, LineNumber, Nimage); vtable = &db[0].vtable; Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/relastro.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/relastro.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/relastro.c (revision 33415) @@ -23,4 +23,10 @@ if (FIT_TARGET == TARGET_HIGH_SPEED) { high_speed_catalogs (); + exit (0); + } + + /* the object analysis is a separate process iterating over catalogs */ + if (FIT_TARGET == TARGET_MERGE_SOURCE) { + relastro_merge_source (); exit (0); } Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/relastro_merge_source.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/relastro_merge_source.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/relastro_merge_source.c (revision 33415) @@ -0,0 +1,138 @@ +# include "relastro.h" + +int relastro_merge_source () { + + int i, m; + SkyTable *sky = NULL; + Catalog catalog_src, catalog_dst; + + // relastro -merge-source (objID) (catID) into (objID) (catID) + // merge the detections of the first source with those of the second source + + // tell libdvo the CATDIR + dvo_set_catdir(CATDIR); + + // load the current sky table (layout of all SkyRegions) + sky = SkyTableLoadOptimal (CATDIR, NULL, NULL, FALSE, SKY_DEPTH, VERBOSE); + SkyTableSetFilenames (sky, CATDIR, "cpt"); + + // we are merging the detections of the src into the dst : OBJ_ID_SRC,CAT_ID_SRC -> OBJ_ID_DST,CAT_ID_DST + + // find the catalog containing the src object: + dvo_catalog_init (&catalog_src, TRUE); + dvo_catalog_init (&catalog_dst, TRUE); + SkyRegion *region_src = NULL; + SkyRegion *region_dst = NULL; + + // load data from each region file, only use bright stars + for (i = 0; i < sky[0].Nregions; i++) { + if (sky[0].regions[i].index == CAT_ID_SRC) { + catalog_src.filename = sky[0].filename[i]; + region_src = &sky[0].regions[i]; + } + if (sky[0].regions[i].index == CAT_ID_DST) { + catalog_dst.filename = sky[0].filename[i]; + region_dst = &sky[0].regions[i]; + } + } + + if (!catalog_src.filename) { + fprintf (stderr, "ERROR: cannot find catalog file matching source catID %x\n", CAT_ID_SRC); + exit (2); + } + if (!catalog_dst.filename) { + fprintf (stderr, "ERROR: cannot find catalog file matching destination catID %x\n", CAT_ID_DST); + exit (2); + } + + // load the source object catalog: + catalog_src.catformat = dvo_catalog_catformat (CATFORMAT); // set the default catformat from config data + catalog_src.catmode = dvo_catalog_catmode (CATMODE); // set the default catmode from config data + catalog_src.catflags = LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF; + catalog_src.Nsecfilt = GetPhotcodeNsecfilt (); + + if (!dvo_catalog_open (&catalog_src, region_src, VERBOSE, "w")) { + fprintf (stderr, "ERROR: failure reading catalog %s\n", catalog_src.filename); + exit (1); + } + if (!catalog_src.Naves_disk) { + fprintf (stderr, "no data in %s, error in cat ID?\n", catalog_src.filename); + exit (1); + } + + // find the source object (by objID) + int index_src = -1; + for (i = 0; i < catalog_src.Naverage; i++) { + if (OBJ_ID_SRC != catalog_src.average[i].objID) continue; + index_src = i; + break; + } + if (index_src < 0) { + fprintf (stderr, "cannot find source object ID %x\n", OBJ_ID_SRC); + exit (1); + } + + // save the measures from this object + // Measure *measures_src; + // ALLOCATE (measures_src, Measure, catalog_src.average[index_src].Nmeasure); + // m = catalog_src.average[index_src].measureOffset; + // for (i = 0; i < catalog_src.average[index_src].Nmeasure; i++, m++) { + // memcpy (&measures_src[i], &catalog_src.measure[m], sizeof(Measure)); + // } + + if (CAT_ID_SRC == CAT_ID_DST) { + + // find the target objects + int index_dst = -1; + for (i = 0; i < catalog_src.Naverage; i++) { + if (OBJ_ID_DST != catalog_src.average[i].objID) continue; + index_dst = i; + break; + } + if (index_dst < 0) { + fprintf (stderr, "cannot find source object ID %x\n", OBJ_ID_DST); + exit (1); + } + + // repoint the src measures at this object + Measure *measures_src; + ALLOCATE (measures_src, Measure, catalog_src.average[index_src].Nmeasure); + m = catalog_src.average[index_src].measureOffset; + for (i = 0; i < catalog_src.average[index_src].Nmeasure; i++, m++) { + + // update objID & catID to match the new source + catalog_src.measure[m].objID = catalog_src.average[index_dst].objID; + catalog_src.measure[m].catID = catalog_src.average[index_dst].catID; + catalog_src.measure[m].averef = index_dst; + + // get the instantaneous positions: + double R = catalog_src.average[index_src].R - catalog_src.measure[m].dR / 3600.0; + double D = catalog_src.average[index_src].D - catalog_src.measure[m].dD / 3600.0; + + // update the offset coordinates to match the new source + catalog_src.measure[m].dR = 3600.0*(catalog_src.average[index_dst].R - R); + catalog_src.measure[m].dD = 3600.0*(catalog_src.average[index_dst].D - D); + } + + // update the count? (updated by resort?) + catalog_src.average[index_dst].Nmeasure += catalog_src.average[index_src].Nmeasure; + + // for the moment, don't delete this object... + catalog_src.average[index_src].Nmeasure = 0; + catalog_src.average[index_src].measureOffset = 0; + + catalog_src.sorted = FALSE; + + // resort the measure table and save + resort_catalog (&catalog_src); + + dvo_catalog_save (&catalog_src, VERBOSE); + dvo_catalog_unlock (&catalog_src); + dvo_catalog_free (&catalog_src); + + exit (0); + } + + return TRUE; +} + Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/resort_catalog.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/resort_catalog.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/resort_catalog.c (revision 33415) @@ -0,0 +1,149 @@ +# include "relastro.h" + +# define myAbort(MSG) { fprintf (stderr, "%s\n", MSG); abort(); } +# define myAssert(LOGIC,MSG) { if (!(LOGIC)) { fprintf (stderr, "%s\n", MSG); abort(); } } + +// sort the measure Sequence based on the average Sequence entries +void SortAveMeasMatch (off_t *MEAS, off_t *AVE, off_t N) { + +# define SWAPFUNC(A,B){ off_t tmp_meas; off_t tmp_ave; \ + tmp_meas = MEAS[A]; MEAS[A] = MEAS[B]; MEAS[B] = tmp_meas; \ + tmp_ave = AVE[A]; AVE[A] = AVE[B]; AVE[B] = tmp_ave; \ + } +# define COMPARE(A,B)(AVE[A] < AVE[B]) + OHANA_SORT (N, COMPARE, SWAPFUNC); +# undef SWAPFUNC +# undef COMPARE +} + +// XXX : where is the time going? perhaps the ALLOCATE? +// XXX : I don't thnk his is getting the right answer yet. + +void resort_catalog (Catalog *catalog) { + + off_t Naverage, Nmeasure; + Measure *measure; + Average *average; + off_t i, j, N, currentAve; + + off_t *measureSeq = NULL; + off_t *averageSeq = NULL; + Measure *measureTMP = NULL; + + if (catalog[0].sorted == TRUE) return; + + // struct timeval start, stop; + // gettimeofday (&start, NULL); + + /* internal counters */ + Nmeasure = catalog[0].Nmeasure; + Naverage = catalog[0].Naverage; + + measure = catalog[0].measure; + average = catalog[0].average; + + // we have a table of average objects and an unsorted table of measurements. each measurement + // has a reference to the average object sequence (as well as an ID) + // measure[i].averef -> average[averef] + // measure[i].objID = average[averef].objID + // measure[i].catID = average[averef].catID + + // we want a sorted measure array with all averef entries in sequence + + ALLOCATE (measureSeq, off_t, Nmeasure); + ALLOCATE (averageSeq, off_t, Nmeasure); + + for (i = 0; i < Nmeasure; i++) { + measureSeq[i] = i; + averageSeq[i] = measure[i].averef; + + if (catalog[0].catformat >= DVO_FORMAT_PS1_V1) { + // earlier formats did not carry the objID or catID, so they are not available (we could assign on load, but we don't) + myAssert(average[averageSeq[i]].objID == measure[measureSeq[i]].objID, "object / detection mismatch"); + myAssert(average[averageSeq[i]].catID == measure[measureSeq[i]].catID, "object / detection mismatch"); + } + } + + // check that averageSeq is now in order + // for (i = 1; i < Nmeasure; i++) { + // if (averageSeq[i] < averageSeq[i-1]) { + // fprintf (stderr, "%d ", (int) i); + // } + // } + // fprintf (stderr, "\n"); + + SortAveMeasMatch(measureSeq, averageSeq, Nmeasure); + // MARKTIME("sort : %f sec\n", dtime); + + // check that averageSeq is now in order + // for (i = 1; i < Nmeasure; i++) { + // if (averageSeq[i] < averageSeq[i-1]) { + // fprintf (stderr, "%d ", (int) i); + // } + // } + // fprintf (stderr, "\n"); + + // copy the measurements in the sorted order + ALLOCATE (measureTMP, Measure, Nmeasure); + for (i = 0; i < Nmeasure; i++) { + j = measureSeq[i]; + measureTMP[i] = measure[j]; + } + // MARKTIME("assign measure : %f sec\n", dtime); + + // update the values of average.measureOffset and average.Nmeasure + FREE(measure); + catalog[0].measure = measureTMP; + + N = 0; + currentAve = averageSeq[0]; + average[currentAve].measureOffset = 0; + for (i = 0; i < Nmeasure; i++) { + if (averageSeq[i] != currentAve) { + // we have hit the next entry in the list + average[currentAve].Nmeasure = N; + N = 0; + currentAve = averageSeq[i]; + average[currentAve].measureOffset = i; + } + N++; + } + // N++; + average[currentAve].Nmeasure = N; + // MARKTIME("update Nmeasure : %f sec\n", dtime); + + int NmeasureTotal = 0; + int measureOffsetOK = TRUE; + for (i = 0; i < catalog[0].Naverage; i++) { + NmeasureTotal += catalog[0].average[i].Nmeasure; + if (VERBOSE && !(NmeasureTotal <= catalog[0].Nmeasure)) { + fprintf (stderr, "too few measurements: %d %d %d\n", (int) i, NmeasureTotal, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset < catalog[0].Nmeasure); + if (VERBOSE && !(catalog[0].average[i].measureOffset < catalog[0].Nmeasure)) { + fprintf (stderr, "offset too large: %d %d %d\n", (int) i, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + measureOffsetOK &= (catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure); + if (VERBOSE && !(catalog[0].average[i].measureOffset + catalog[0].average[i].Nmeasure <= catalog[0].Nmeasure)) { + fprintf (stderr, "orrset + Nmeasure too large: %d + %d > %d %d\n", (int) i, catalog[0].average[i].measureOffset, catalog[0].average[i].Nmeasure, (int) catalog[0].Nmeasure); + } + } + + if (!measureOffsetOK) { + fprintf (stderr, "ERROR: catalog %s has an invalid measureOffset\n", catalog[0].filename); + } + + if (NmeasureTotal != catalog[0].Nmeasure) { + fprintf (stderr, "ERROR: catalog %s has an invalid Nmeasure\n", catalog[0].filename); + } + + // MARKTIME(" match time %9.4f sec for %7lld measures, %6lld average\n", dtime, (long long) Nmeasure, (long long) Naverage); + + catalog[0].sorted = TRUE; + + FREE (measureSeq); + FREE (averageSeq); + + return; +} + Index: branches/meh_branches/ppstack_test/Ohana/src/relastro/src/testparallax.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relastro/src/testparallax.c (revision 33415) +++ branches/meh_branches/ppstack_test/Ohana/src/relastro/src/testparallax.c (revision 33415) @@ -0,0 +1,31 @@ +# include "relastro.h" + +int main (int argc, char **argv) { + + if (argc != N) { + fprintf (stderr, "USAGE: testparallax (file.dat)\n"); + exit (1); + } + + // test parallax program + FILE *f = fopen (argv[1], "r"); + if (!f) { fprintf (stderr, "failed to open %s\n", argv[1]); exit (2); } + + char buffer[1024]; + scan_line(f, line); + + while (scan_line(f, line) != EOF) { + dparse (&jd, 1, line); + dparse (&time, 2, line); + dparse (&ra, 3, line); + dparse (&dec, 4, line); + + } + + // project to local coords + + // run fitter + FitPMandPar + + +} Index: branches/meh_branches/ppstack_test/Ohana/src/relphot/src/load_images.c =================================================================== --- branches/meh_branches/ppstack_test/Ohana/src/relphot/src/load_images.c (revision 32161) +++ branches/meh_branches/ppstack_test/Ohana/src/relphot/src/load_images.c (revision 33415) @@ -85,14 +85,4 @@ } - - // generate db->vtable from db->ftable based on the selection - // XXX does this simply duplicate the memory needlessly? we recreate these lines - // in reload_images. If we had saved the line numbers, we could avoid this - // vtable points *another* copy of the subset rows - // (the later call to 'reload_images' copies the subset elements back on top of - // the rows of the vtable) - // gfits_vtable_from_ftable (&db[0].ftable, &db[0].vtable, LineNumber, Nsubset); - // MARKTIME("converted ftable to vtable: %f sec\n", dtime); - // save the subset of images in the static reference in ImageOps, set up indexes initImages (subset, LineNumber, Nsubset); Index: branches/meh_branches/ppstack_test/PS-IPP-Config/lib/PS/IPP/Config.pm =================================================================== --- branches/meh_branches/ppstack_test/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 32161) +++ branches/meh_branches/ppstack_test/PS-IPP-Config/lib/PS/IPP/Config.pm (revision 33415) @@ -20,4 +20,5 @@ use Getopt::Long 2.33 qw( GetOptions :config gnu_getopt pass_through ); # Set pass_through so we don't kill @ARGV use IPC::Cmd 0.36 qw( can_run run ); +use POSIX qw( strftime ); use base qw( Exporter Class::Accessor::Fast ); @@ -397,5 +398,8 @@ return $self->{nebulous} if defined $self->{nebulous}; # Already started - my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server + # For consistency with the rest of the IPP, don't use site.config + # to set NEB_SERVER use environment variable. + # my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server + my $server = $ENV{NEB_SERVER}; unless (defined $server) { carp "Unable to find NEB_SERVER in camera configuration file."; @@ -692,4 +696,6 @@ } + + # redirect stderr and stdout streams to a file sub redirect_output @@ -722,4 +728,43 @@ return 1; +} + +# redirect stderr and stdout streams to log file. If file already exists rename it and record +# the name in the new log file. +sub redirect_to_logfile { + my $self = shift; + my $name = shift; + + my $tstring = strftime "%Y%m%dT%H%M%S", gmtime; + + my $old_name; + my $scheme = file_scheme($name); + if ($scheme and $scheme eq 'neb') { + my $neb = $self->nebulous; + if ($neb->storage_object_exists($name)) { + eval { + $old_name = "$name.$tstring"; + $neb->move($name, $old_name); + }; + if ($@) { + carp "nebulous move failed for $name"; + return undef; + } + } + } else { + if ($self->file_exists($name)) { + my $resolved = $self->file_resolve($name); + $old_name = "$resolved.$tstring"; + if (!rename($resolved, $old_name)) { + carp "Failed to rename $resolved $old_name"; + return undef; + } + } + } + my $result = $self->redirect_output($name); + if ($result and $old_name) { + print "\nPrevious log file moved to $old_name\n"; + } + return $result; } Index: branches/meh_branches/ppstack_test/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm =================================================================== --- branches/meh_branches/ppstack_test/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 32161) +++ branches/meh_branches/ppstack_test/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (revision 33415) @@ -238,5 +238,5 @@ # choose the chips containing the center pixel by calling selectComponets() below if ($component or $use_imfile_id or !$skycenter) { - $command = "$chiptool -processedimfile -dbname $imagedb"; + $command = "$chiptool -processedimfile -pstamp_order -dbname $imagedb"; if ($component and $component ne 'all') { $class_id = $component; @@ -263,5 +263,5 @@ } elsif ($stage eq "warp") { if ($component or $use_imfile_id or !$skycenter) { - $command = "$warptool -warped -dbname $imagedb"; + $command = "$warptool -warped -pstamp_order -dbname $imagedb"; if ($component and $component ne 'all') { $skycell_id = $component; @@ -281,5 +281,5 @@ } elsif ($stage eq "diff") { if ($component or $use_imfile_id or !$skycenter) { - $command = "$difftool -diffskyfile -dbname $imagedb"; + $command = "$difftool -diffskyfile -pstamp_order -dbname $imagedb"; if ($component and $component ne 'all') { $skycell_id = $component; @@ -302,5 +302,5 @@ $command = "$stacktool -sumskyfile -dbname $imagedb"; $id_opt = "-stack_id"; - $component_args = " -skycell_id $skycell_id" if $skycell_id; + $component_args = " -skycell_id $skycell_id" if $skycell_id and $skycell_id ne 'all'; if (!$unconvolved) { @@ -351,5 +351,5 @@ $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin; $command .= " -dateobs_end $dateobs_end" if $dateobs_end; - } else { + } elsif ($req_type ne "byid") { $command .= " -mjd_obs_begin $mjd_min" if $mjd_min; $command .= " -mjd_obs_end $mjd_max" if $mjd_max; @@ -438,4 +438,7 @@ } elsif ($stage eq "stack") { $stage_id = $out->{stack_id}; + # dquery wants these set + $out->{magicked} = 0; + $out->{data_state} = $out->{state}; } $out->{stage_id} = $stage_id; @@ -487,10 +490,10 @@ my $command = "$difftool -dbname $imagedb -pstamp_order"; - my $listrun = 0; + my $choose_components = 0; if ($byid) { if ($skycell_id and ($skycell_id ne 'all')) { $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id"; } else { - $listrun = 1; + $choose_components = 1; $command .= " -listrun -diff_id $id"; # the following is a work around for the problem reported in ticket #1394 @@ -512,5 +515,5 @@ my $images; - if ($listrun) { + if ($choose_components) { $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose); } else { @@ -603,5 +606,5 @@ my @imageList = ($image); - setRowRefs($rowList, \@imageList); + setRowRefs($rowList, \@imageList) unless $choose_components; # the $image is going to be returned directly in this case so we need to duplicate # some of processing that lookup does for other img_types @@ -847,4 +850,5 @@ { my $command = "$whichimage $ra $dec"; + $command .= " --tess_id $requested_tess_id" if $requested_tess_id; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); Index: branches/meh_branches/ppstack_test/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm =================================================================== --- branches/meh_branches/ppstack_test/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 32161) +++ branches/meh_branches/ppstack_test/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (revision 33415) @@ -27,6 +27,8 @@ $PSTAMP_SELECT_PSF $PSTAMP_SELECT_BACKMDL + $PSTAMP_SELECT_JPEG $PSTAMP_SELECT_INVERSE $PSTAMP_SELECT_UNCONV + $PSTAMP_RESTORE_BACKGROUND $PSTAMP_REQUEST_UNCENSORED $PSTAMP_REQUIRE_UNCENSORED @@ -63,6 +65,8 @@ our $PSTAMP_SELECT_PSF = 16; our $PSTAMP_SELECT_BACKMDL = 32; +our $PSTAMP_SELECT_JPEG = 64; our $PSTAMP_SELECT_INVERSE = 1024; our $PSTAMP_SELECT_UNCONV = 2048; +our $PSTAMP_RESTORE_BACKGROUND = 4096; our $PSTAMP_USE_IMFILE_ID = 16384; Index: branches/meh_branches/ppstack_test/dbconfig/add.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/add.md (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/add.md (revision 33415) @@ -3,5 +3,6 @@ add_id S64 0 # Primary Key AUTO_INCREMENT stage STR 64 # what the stage is (warp, cam, diff, - stage_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + stage_id S64 0 # Key INDEX(add_id,cam_id) fkey(cam_id) ref camRun(cam_id) + stage_extra1 S32 0 state STR 64 # Key workdir STR 255 Index: branches/meh_branches/ppstack_test/dbconfig/changes.txt =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/changes.txt (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/changes.txt (revision 33415) @@ -2135,2 +2135,122 @@ alter table addRun add key(stage), add key (stage_id); + +CREATE TABLE vpRun ( + vp_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + workdir VARCHAR(255), + note VARCHAR(255), + dest_id BIGINT, + outroot VARCHAR(255), + dtime_script FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(vp_id), + KEY(exp_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(fault), + INDEX(vp_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpProcessedCell ( + vp_id BIGINT, + class_id VARCHAR(64), + cell_id VARCHAR(64), + dtime_photom FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + path_base VARCHAR(255), + fault SMALLINT NOT NULL DEFAULT 0, + KEY(quality), + KEY(fault), + FOREIGN KEY(vp_id) REFERENCES vpRun(vp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- changes to prevent repeatedly queueing diffRuns for warps with no overlaps +ALTER TABLE diffInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(diff_skyfile_id); +ALTER TABLE diffInputSkyfile CHANGE COLUMN skycell_id skycell_id VARCHAR(64) DEFAULT NULL; + +UPDATE dbversion set schema_version = '1.1.71', updated= CURRENT_TIMESTAMP(); + + +-- this is to stuff extra stage related information. +-- example - multi filter staticsky stage - this allows selection of +-- each filter of cmf staticsky. + +alter table addRun add column stage_extra1 int after stage_id; + + +-- This adds new features to addRun, first add the tables: + +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +alter table mergedvodbProcessed add column merge_order BIGINT after merge_id; + +-- then copy over the info into the 2 tables: + insert into mergedvodbRun ( minidvodb_id, mergedvodb,mergedvodb_path, + state, creation_date) select minidvodb_id, minidvodb_group,mergedvodb_path, +minidvodbRun.state, + minidvodbRun.creation_date from minidvodbRun where minidvodb_id >= 0; + +insert into mergedvodbProcessed (merge_id, merge_order, dtime_verify, +dtime_merge, dtime_script, epoch, fault) select merge_id, merge_order, +minidvodbProcessed.dtime_verify, minidvodbProcessed.dtime_merge, +minidvodbProcessed.dtime_script, minidvodbProcessed.epoch, +minidvodbProcessed.fault from minidvodbProcessed join mergedvodbRun using +(minidvodb_id) where merge_id > -1 ; + +--then clean up the tables: +-- once the tables have had the columns migrated over, then and only then do this: + +alter table minidvodbRun drop column mergedvodb_path; +alter table minidvodbProcessed drop column dtime_verify; +alter table minidvodbProcessed drop column merge_order; +alter table minidvodbProcessed drop column dtime_merge; +alter table minidvodbProcessed drop column mergedvodb_path; + + +--then to keep compatible with old camruns (stage_extra1 = negative means it used a destreaked smf) + +--it's commented out as it only needs to be run once (and only on gpc1) +--update addRun set stage_extra1 = -1 where stage = 'cam' <= 483624; + Index: branches/meh_branches/ppstack_test/dbconfig/config.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/config.md (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/config.md (revision 33415) @@ -2,4 +2,4 @@ pkg_name STR ippdb pkg_namespace STR ippdb - pkg_version STR 1.1.66 + pkg_version STR 1.1.71717171717171 END Index: branches/meh_branches/ppstack_test/dbconfig/ipp.m4 =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/ipp.m4 (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/ipp.m4 (revision 33415) @@ -25,4 +25,5 @@ include(magic.md) include(minidvodb.md) +include(mergedvodb.md) include(calibration.md) include(flatcorr.md) @@ -36,2 +37,3 @@ include(diffphot.md) include(lap.md) +include(vp.md) Index: branches/meh_branches/ppstack_test/dbconfig/mergedvodb.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/mergedvodb.md (revision 33415) +++ branches/meh_branches/ppstack_test/dbconfig/mergedvodb.md (revision 33415) @@ -0,0 +1,29 @@ +mergedvodbRun METADATA + merge_id S64 0 + minidvodb_id S64 0 + mergedvodb STR 64 + mergedvodb_path STR 255 + state STR 64 + creation_date UTC 0001-01-01T00:00:00Z +END + +mergedvodbProcessed METADATA + merge_id S64 0 + merge_order S64 0 + dtime_verify F32 0.0 + dtime_merge F32 0.0 + dtime_script F32 0.0 + epoch UTC 0001-01-01T00:00:00Z + fault S16 0 +END + +mergedvodbCopy METADATA + mergevodbcopy_id S64 0 + merge_id S64 0 + mergedvodb_rsync_path STR 255 + destination_host STR 255 + fault S16 0 + state STR 64 + epoch UTC 0001-01-01T00:00:00Z + dtime F32 0.0 +END Index: branches/meh_branches/ppstack_test/dbconfig/minidvodb.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/minidvodb.md (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/minidvodb.md (revision 33415) @@ -4,5 +4,4 @@ minidvodb_group STR 64 minidvodb_path STR 255 - mergedvodb_path STR 255 state STR 64 creation_date UTC 0001-01-01T00:00:00Z @@ -11,12 +10,8 @@ minidvodbProcessed METADATA minidvodb_id S64 0 - merge_order S64 0 dtime_resort F32 0.0 dtime_relphot F32 0.0 - dtime_verify F32 0.0 - dtime_merge F32 0.0 dtime_script F32 0.0 epoch UTC 0001-01-01T00:00:00Z - mergedvodb_path STR 255 fault S16 0 END Index: branches/meh_branches/ppstack_test/dbconfig/pstamp.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/pstamp.md (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/pstamp.md (revision 33415) @@ -48,4 +48,6 @@ dep_id S64 0 fault_count S32 0 + parent_id S64 0 + is_parent BOOL f END @@ -67,2 +69,8 @@ num S64 0 # Primary Key AUTO_INCREMENT END + +pstampFile METADATA + file_id S64 0 # Primary Key AUTO_INCREMENT + job_id S64 0 + path STR 255 +end Index: branches/meh_branches/ppstack_test/dbconfig/staticsky.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/staticsky.md (revision 32161) +++ branches/meh_branches/ppstack_test/dbconfig/staticsky.md (revision 33415) @@ -31,2 +31,32 @@ quality S16 0 # Key END + +skycalRun METADATA + skycal_id S64 0 # Primary Key AUTO_INCREMENT + sky_id S64 0 # fkey(sky_id) ref staticskyRun(sky_id) + stack_id S64 0 # fkey(stack_id) ref stackRun(stack_id) + state STR 64 # Key + workdir STR 255 + label STR 64 # Key + data_group STR 64 # Key + dist_group STR 64 # Key + reduction STR 64 # Reduction class + registered TAI NULL + note STR 255 +END + +skycalResult METADATA + skycal_id S64 0 # Primary Key fkey(skycal_id) ref skycalRun(skycal_id) + path_base STR 255 + dtime_script F32 0.0 + dtime_astrom F32 0.0 + sigma_ra F32 0.0 + sigma_dec F32 0.0 + n_astrom S32 0 + zpt_obs F32 0.0 + zpt_stdev F32 0.0 + quality S16 0 # Key + software_ver STR 16 + hostname STR 64 + fault S16 0 # Key +END Index: branches/meh_branches/ppstack_test/dbconfig/vp.md =================================================================== --- branches/meh_branches/ppstack_test/dbconfig/vp.md (revision 33415) +++ branches/meh_branches/ppstack_test/dbconfig/vp.md (revision 33415) @@ -0,0 +1,24 @@ +vpRun METADATA + vp_id S64 0 # Primary Key AUTO_INCREMENT + exp_id S64 64 # Key INDEX(vp_id, exp_id) fkey (exp_id) ref rawExp(exp_id) + state STR 64 # Key + label STR 64 # Key + data_group STR 64 # Key + workdir STR 255 + note STR 255 + dest_id S64 0 + outroot STR 255 + dtime_script F32 0.0 + hostname STR 64 + fault S16 0 # Key NOT NULL +END + +vpProcessedCell METADATA + vp_id S64 0 # Primary Key fkey (vp_id) ref vpRun(vp_id) + class_id STR 64 # Primary Key + cell_id STR 64 # Primary Key + dtime_photom F32 0.0 + quality S16 0 + path_base STR 255 + fault S16 0 # Key NOT NULL +END Index: branches/meh_branches/ppstack_test/ippMonitor/INSTALL =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/INSTALL (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/INSTALL (revision 33415) @@ -13,12 +13,6 @@ psconfigure --dbhost alala --dbuser ipp --dbpass ipp --htdocs /var/www/kiawe -ipp004 / ipp017 example: - psconfigure --dbhost ipp017 --dbuser ipp --dbpass ipp --htdocs ~ipp/htdocs +# note that we have a special version for ippmonitor without $PSCONFIG/$PSCONFDIR values: +ipp@ippdb03 / ippdb03 example: + psconfigure --dbhost ippdb01 --dbuser ipp --dbpass ipp --htdocs /var/www/localhost/htdocs --site ~/ippconfig/ippmonitor.config -ipp004 / ipp004 example: - psconfigure --dbhost ipp004 --dbuser ipp --dbpass ipp --htdocs ~ipp/htdocs - -# note that we have a special version for ippmonitor without $PSCONFIG/$PSCONFDIR values: -ipp@ipp004 / ippdb01 example: - psconfigure --dbhost ippdb01 --dbuser ipp --dbpass ipp --htdocs ~/htdocs --site ~/ippconfig/ippmonitor.config - Index: branches/meh_branches/ppstack_test/ippMonitor/Makefile.in =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/Makefile.in (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/Makefile.in (revision 33415) @@ -60,4 +60,5 @@ $(DESTWWW)/czartool_getplot.php \ $(DESTWWW)/czartool_exposures.php \ +$(DESTWWW)/czartool_ipptopsps.php \ $(DESTWWW)/histogram.php \ $(DESTWWW)/show_and_delete_image.php \ Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czarplot.pl =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czarplot.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czarplot.pl (revision 33415) @@ -25,10 +25,9 @@ my $magicMask = undef; my $rateInterval = undef; -my $deriv = undef; my $nebulous = undef; my $savingToFile = undef; -my $analysis = undef; my $exposureId = undef; my $log = undef; +my $psps = undef; GetOptions ( @@ -46,9 +45,8 @@ "nebulous|n" => \$nebulous, "rate|r" => \$rate, - "deriv|f" => \$deriv, - "analysis|a" => \$analysis, "timeseries|t" => \$timeSeries, "verbose|v" => \$verbose, "log|g" => \$log, + "psps|u" => \$psps, ); @@ -60,7 +58,4 @@ print "* UNKNKOWN: option @ARGV\n"; } -if ($analysis && !$stage) { - $quit = 1; - print "* REQUIRED: choose a stage for analsis -s (default=none)\n";} if (!$histogram) { print "* OPTIONAL: plot histogram -h (default=off)\n";} @@ -73,10 +68,4 @@ if (!$rate) { print "* OPTIONAL: plot histogram of rate -r (default=off)\n";} -if (!$deriv) { - $deriv = 0; - print "* OPTIONAL: plot first derivative -f (default=$deriv)\n";} -if (!$analysis) { - $analysis = 0; - print "* OPTIONAL: include analysis -a (default=$analysis)\n";} if (!$log) { $log = 0; @@ -94,5 +83,5 @@ print "* OPTIONAL: time interval for rate plot -i <'1 hour'|'1 day'|etc> (default=depends on time frame)\n";} if (!$begin) { - print "* OPTIONAL: choose a begin time -b (default=6:35am this morning)\n";} + print "* OPTIONAL: choose a begin time -b (default=18:00)\n";} if (!$end) { print "* OPTIONAL: choose an end time -e (default=now)\n";} @@ -102,4 +91,6 @@ print "* OPTIONAL: choose an output location -o (default=none)\n"; } +if (!$psps) { + print "* OPTIONAL: plot ippToPsps stages -u (default=off)\n";} print "*\n*******************************************************************************\n"; @@ -109,5 +100,5 @@ # default values -if (!$rate && !$magicMask && !$nebulous && !$histogram && !$analysis && !$timeSeries) {$timeSeries = 1;} +if (!$rate && !$magicMask && !$nebulous && !$histogram && !$timeSeries) {$timeSeries = 1;} if (!$verbose) {$verbose = 0;} if (!$save_temps) {$save_temps = 0;} @@ -128,4 +119,6 @@ $plotter = czartool::Plotter->new_display($config, $save_temps); } + +if ($psps) {$plotter->setIppToPspsMode();} # if a single day has been chosen @@ -182,7 +175,2 @@ } -if($analysis) { - - my $stageMetrics = new czartool::StageMetrics($stage, $label, $begin, $end); - if ($czarDb->runAnalysis($stageMetrics)) {$stageMetrics->printMe();} -} Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czarpoll.pl =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czarpoll.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czarpoll.pl (revision 33415) @@ -37,6 +37,28 @@ -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); - +my @stages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + +my @ippToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + +my @allStages = (@stages, @ippToPspsStages); timePoll($period); @@ -219,5 +241,5 @@ updateAllStages($thisServer, $newState, $labels, $begin, $end); - createPlots($thisServer, $labels, $begin, $end); + if ($thisServer eq "stdscience") {createPlots($thisServer, $labels, $begin, $end);} } else { print "* WARNING: no $thisServer labels found in Db\n";} @@ -244,5 +266,29 @@ print "* Generating plots\n"; + # get ippToPsps labels + my $ippToPspsLabels = undef; + $czarDb->getCurrentIppToPspsLabels($begin, $end, \$ippToPspsLabels); + $plotter->setIppToPspsMode(); + + # create ippToPsps labels + foreach $stage (@ippToPspsStages) { + foreach $row ( @{$ippToPspsLabels} ) { + my ($label) = @{$row}; + + chomp($label); + $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1); + } + } + + # create plots for each label for all stages for ippToPsps + foreach $row ( @{$ippToPspsLabels} ) { + my ($label) = @{$row}; + + $plotter->createTimeSeries($label, undef, $begin, $end, 1, 1, 1); + } + + # create plots for each label for each stage + $plotter->setStandardMode(); foreach $stage (@stages) { foreach $row ( @{$rows} ) { @@ -250,5 +296,5 @@ chomp($label); - $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 1, 1); + $plotter->createTimeSeries($label, $stage, $begin, $end, 1, 0, 1); } } @@ -265,9 +311,9 @@ $allServerLabels = "all_".$server."_labels"; - $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 1, 1); + $plotter->createTimeSeries($allServerLabels, undef, $begin, $end, 1, 0, 1); foreach $stage (@stages) { - $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 1, 1); # TODO must be a neater way... + $plotter->createTimeSeries($allServerLabels, $stage, $begin, $end, 1, 0, 1); # TODO must be a neater way... } } Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/CzarDb.pm =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/CzarDb.pm (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/CzarDb.pm (revision 33415) @@ -8,5 +8,28 @@ use File::Temp; -my @stages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); # TODO put elsewhere +my @stages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); # TODO put elsewhere + +my @ippToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + +my @allStages = (@stages, @ippToPspsStages); use base 'czartool::MySQLDb'; @@ -28,5 +51,4 @@ bless $self, $class; - $self->update(); return $self; } @@ -34,5 +56,28 @@ ########################################################################### # -# Gets current_labels table +# Gets current ippToPsps labels +# +########################################################################### +sub getCurrentIppToPspsLabels { + my ($self, $begin, $end, $labels) = @_; + + my $query = $self->{_db}->prepare(<= '$begin' + AND timestamp <= '$end' +SQL + + if (!$query->execute) {return 0;} + + ${$labels} = $query->fetchall_arrayref(); + + return 1; +} + +########################################################################### +# +# Gets current labels from the current_labels table # ########################################################################### @@ -673,4 +718,6 @@ # times $timeSep = $self->diffTimesInSecs($thisTimestamp, $lastTimestamp); + if ($timeSep == 0) {$timeSep = 1;}; # HACK to prevent divide-by-zero + $middleTime = $self->getMiddleTime($lastTimestamp, $thisTimestamp); $formattedMiddleTimestamp = $self->getFormattedDate($middleTime); @@ -1108,5 +1155,5 @@ my $stage = undef; $totalDeleted = 0; - foreach $stage (@stages) { + foreach $stage (@allStages) { if (!$self->getLabelsInThisTimePeriod($stage, $fromTime, $toTime, \$labels)) {next;} @@ -1214,5 +1261,5 @@ my $stage = undef; - foreach $stage (@stages) { + foreach $stage (@allStages) { $self->optimizeTable($stage); @@ -1342,432 +1389,4 @@ } -########################################################################### -# -# Update Db to newest version -# -########################################################################### -sub update { - my ($self) = @_; - - my $currentRevision = -1; - - while (1) { - - $currentRevision = $self->getRevision(); - if ($self->{_verbose}) {print "* Current Db revision = $currentRevision\n";} - - if ($currentRevision == 0) {$self->createRevision_1();} - elsif ($currentRevision == 1) {$self->createRevision_2();} - elsif ($currentRevision == 2) {$self->createRevision_3();} - elsif ($currentRevision == 3) {$self->createRevision_4();} - elsif ($currentRevision == 4) {$self->createRevision_5();} - elsif ($currentRevision == 5) {$self->createRevision_6();} - elsif ($currentRevision == 6) {$self->createRevision_7();} - elsif ($currentRevision == 7) {$self->createRevision_8();} - elsif ($currentRevision == 8) {$self->createRevision_9();} - elsif ($currentRevision == 9) {$self->createRevision_10();} - elsif ($currentRevision == 10) {$self->createRevision_11();} - elsif ($currentRevision == 11) {$self->createRevision_12();} - elsif ($currentRevision == 12) {$self->createRevision_13();} - else {last;} - } -} - -####################################################################################### -# -# Sets current revision of ippToPsps database -# -####################################################################################### -sub setRevision { - my ($self, $revision) = @_; - - my $query = $self->{_db}->prepare(<execute; -} - -####################################################################################### -# -# Gets current revision of ippToPsps database -# -####################################################################################### -sub getRevision { - my ($self) = @_; - - if (!$self->SUPER::doesTableExist("revision")) {return 0;} - - my $query = $self->{_db}->prepare(<execute; - my @row = $query->fetchrow_array(); - - return $row[0]; -} - -####################################################################################### -# -# Create revision 1 of the database -# -####################################################################################### -sub createRevision_1 { - my ($self) = @_; - - print "* Creating revision 1 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - my $stage = undef; - foreach $stage (@stages) { - - my $query = $self->{_db}->prepare(<execute; - - } - - $self->setRevision(1); -} - -####################################################################################### -# -# Create revision 2 of the database -# -####################################################################################### -sub createRevision_2 { - my ($self) = @_; - - print "* Creating revision 2 of '$self->{_dbName}'\n"; - - - my $stage = undef; - foreach $stage (@stages) { - - my $query = $self->{_db}->prepare(<execute; - - } - - $self->setRevision(2); -} - -####################################################################################### -# -# Create revision 3 of the database -# -####################################################################################### -sub createRevision_3 { - my ($self) = @_; - - print "* Creating revision 3 of '$self->{_dbName}'\n"; - - - my $stage = undef; - foreach $stage (@stages) { - - my $query = $self->{_db}->prepare(<execute; - } - - $self->setRevision(3); -} - -####################################################################################### -# -# Create revision 4 of the database -# -####################################################################################### -sub createRevision_4 { - my ($self) = @_; - - print "* Creating revision 4 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - $self->setRevision(4); -} - -####################################################################################### -# -# Create revision 5 of the database -# -####################################################################################### -sub createRevision_5 { - my ($self) = @_; - - print "* Creating revision 5 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - $self->setRevision(5); -} - -####################################################################################### -# -# Create revision 6 of the database -# -####################################################################################### -sub createRevision_6 { - my ($self) = @_; - - print "* Creating revision 6 of '$self->{_dbName}'\n"; - - my $stage = undef; - my $index = undef; - foreach $stage (@stages) { - - $index = $stage . "Index"; - my $query = $self->{_db}->prepare(<execute; - } - - my $query = $self->{_db}->prepare(<execute; - - - $self->setRevision(6); -} - -####################################################################################### -# -# Create revision 7 of the database -# -####################################################################################### -sub createRevision_7 { - my ($self) = @_; - - print "* Creating revision 7 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - $self->setRevision(7); -} - -####################################################################################### -# -# Create revision 8 of the database -# -####################################################################################### -sub createRevision_8 { - my ($self) = @_; - - print "* Creating revision 8 of '$self->{_dbName}'\n"; - - # drop reverting column from all stages tables - my $stage = undef; - foreach $stage (@stages) { - - my $query = $self->{_db}->prepare(<execute; - } - - # create new 'revert' table - my $query = $self->{_db}->prepare(<execute; - - # insert stages into revert table - foreach $stage (@stages) { - my $query = $self->{_db}->prepare(<execute; - } - - $self->setRevision(8); -} - -####################################################################################### -# -# Create revision 9 of the database -# -####################################################################################### -sub createRevision_9 { - my ($self) = @_; - - print "* Creating revision 9 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - $query = $self->{_db}->prepare(<execute; - - $query = $self->{_db}->prepare(<execute; - - $self->setRevision(9); -} - -####################################################################################### -# -# Create revision 10 of the database -# -####################################################################################### -sub createRevision_10 { - my ($self) = @_; - - print "* Creating revision 10 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - $self->setRevision(10); -} - -####################################################################################### -# -# Create revision 11 of the database -# -####################################################################################### -sub createRevision_11 { - my ($self) = @_; - - print "* Creating revision 11 of '$self->{_dbName}'\n"; - - # same shape as other stage tables to enable easy update - #my $query = $self->{_db}->prepare(<execute; -# $query = $self->{_db}->prepare(<execute; - - $self->setRevision(11); -} - -####################################################################################### -# -# Create revision 12 of the database -# -# - adding nightlyscience table -# -####################################################################################### -sub createRevision_12 { - my ($self) = @_; - - print "* Creating revision 12 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - $self->setRevision(12); -} -####################################################################################### -# -# Create revision 13 of the database -# -# adding server_dates table -# -####################################################################################### -sub createRevision_13 { - my ($self) = @_; - - print "* Creating revision 13 of '$self->{_dbName}'\n"; - - my $query = $self->{_db}->prepare(<execute; - - $self->setRevision(13); -} 1; Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Gpc1Db.pm =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Gpc1Db.pm (revision 33415) @@ -233,4 +233,14 @@ } + # 'new' stuff really includes 'update' stuff, at least for LAP. This is a hack, but there you go + elsif ($state eq "new") { + + $query = $self->{_db}->prepare(<{htmlFile}; - print $htmlFile "\"master\n"; - print $htmlFile "\"master\n"; + #print $htmlFile "\"master\n"; + #print $htmlFile "\"master\n"; opendir(DIR, $self->{path}) or die $!; Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Pantasks.pm =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Pantasks.pm (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Pantasks.pm (revision 33415) @@ -10,4 +10,5 @@ "addstarlap", "cleanup", + "deepstack", "detrend", "distribution", Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Plotter.pm =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Plotter.pm (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/Plotter.pm (revision 33415) @@ -10,5 +10,27 @@ use czartool::Config; -my @allStages = ("burntool", "chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist", "pub"); +my @allStages = ( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + +my @allIppToPspsStages = ( + "processed", + "loaded_to_datastore", + "loaded_to_ODM", + "merge_worthy", + "deleted_datastore", + "deleted_dxlayer", + "merged", + "deleted_local"); + ########################################################################### @@ -75,4 +97,26 @@ ########################################################################### # +# Sets plotter to use standard IPP mode, i.e. chip, cam etc +# +########################################################################### +sub setStandardMode { + my ($self) = @_; + + $self->{allStages} = \@allStages; +} + +########################################################################### +# +# Sets plotter to use ippToPsps mode, i.e. processed, loaded_to_datastore etc +# +########################################################################### +sub setIppToPspsMode { + my ($self) = @_; + + $self->{allStages} = \@allIppToPspsStages; +} + +########################################################################### +# # Some initialisation code common to all constructors # @@ -83,4 +127,5 @@ $self->{_gpc1Db} = $self->{_config}->getGpc1Instance(); $self->{_czarDb} = $self->{_config}->getCzarDbInstance(); + $self->setStandardMode(); } @@ -140,5 +185,5 @@ my $stages = undef; if (!$selectedStage) { - $stages = \@allStages; + $stages = \@{$self->{allStages}}; $selectedStage = 'all_stages'; } @@ -203,10 +248,10 @@ "set style histogram rowstacked;" . "set style fill solid border -1;" . - "set ylabel \"Exposures processed per $interval\";" . + "set ylabel \"Items processed per $interval\";" . "set boxwidth 0.75;" . "plot "; my $first = 1; - foreach my $stage (@allStages) { + foreach my $stage (@{$self->{allStages}}) { @@ -226,5 +271,5 @@ ########################################################################### # -# Plots a time series for all stages for this label +# Plots a time series for all or one stage for this label # ########################################################################### @@ -238,5 +283,5 @@ my $stages = undef; - if (!$selectedStage) {$stages = \@allStages;} + if (!$selectedStage) {$stages = \@{$self->{allStages}};} else {$stages = ["$selectedStage"]}; @@ -297,5 +342,5 @@ $timeDiff, "", - "Exposures"); + "Items"); } @@ -314,5 +359,5 @@ $timeDiff, "", - "Log(Exposures)"); + "Log(Items)"); } @@ -331,5 +376,5 @@ $timeDiff, "", - "Exposures per hour"); + "Items per hour"); } } @@ -351,5 +396,5 @@ my $stage = undef; my $pendingMinusFaults = undef; - foreach $stage (@allStages) { + foreach $stage (@{$self->{allStages}}) { if (!$self->{_czarDb}->countProcessedPendingAndFaults( @@ -381,5 +426,5 @@ "set style histogram rowstacked;" . "set style fill solid border -1;" . - "set ylabel \"Exposures\";" . + "set ylabel \"Items\";" . "set boxwidth 0.75;" . "plot '".$tmpFile->filename."' using 2:xtic(1) title \"Faults\" lt 1, '' using 3 title \"Processed\" lt 2, '' using 4 title \"Pending\" lt 7;" . @@ -520,5 +565,6 @@ my $firstIn = 1; # loop through stages array so that they are ordered properly (not maintained in hash) - foreach my $stage (@allStages) { + #foreach my $stage (@allStages) { + foreach my $stage (@{$self->{allStages}}) { if(!$gnuplotFiles->{$stage}) {next;} @@ -620,4 +666,5 @@ "set title \"Nebulous disk use across IPP cluster ($totalPercent of total allocated)\";" . "set style fill solid 1.00 border -1;" . + "set key center top;" . "set style histogram rowstacked;" . "set style data histograms;" . Index: branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/czarconfig.xml =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/czarconfig.xml (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/czartool/czartool/czarconfig.xml (revision 33415) @@ -9,5 +9,5 @@ /usr/share/fonts/corefonts/arial.ttf 8 - /data/ipp004.0/czartool_plots + /data/ipp009.0/czartool_plots @@ -22,6 +22,6 @@ gpc1 ippdb01 - ippuser - ippuser + ippMonitor + ippMonitor Index: branches/meh_branches/ppstack_test/ippMonitor/def/chipProcessedImfile.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/chipProcessedImfile.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/chipProcessedImfile.d (revision 33415) @@ -9,4 +9,6 @@ WHERE chipProcessedImfile.chip_id = chipRun.chip_id WHERE chipProcessedImfile.fault = 0 + +#UNRESTRICTED WHERE 0=1 ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: branches/meh_branches/ppstack_test/ippMonitor/def/distRun.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/distRun.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/distRun.d (revision 33415) @@ -7,4 +7,5 @@ WHERE distRun.dist_id = distComponent.dist_id +UNRESTRICTED AND 0=1 # field size format name show link to extras Index: branches/meh_branches/ppstack_test/ippMonitor/def/lapExposures.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/lapExposures.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/lapExposures.d (revision 33415) @@ -6,54 +6,55 @@ UNRESTRICTED WHERE 0=1 -ARGS ARG1 rawImfile.exp_id=$lapExp.exp_id -ARGS ARG2 chipRun.chip_id=$lapExp.chip_id -ARGS ARG3 chipRun.chip_id=$lapExp.pair_id -ARGS ARG4 camRun.cam_id=$camRun1.cam_id -ARGS ARG5 camRun.cam_id=$camRun2.cam_id -ARGS ARG6 fakeRun.fake_id=$fakeRun1.fake_id -ARGS ARG7 fakeRun.fake_id=$fakeRun2.fake_id -ARGS ARG8 warpRun.warp_id=$warpRun1.warp_id -ARGS ARG9 warpRun.warp_id=$warpRun2.warp_id -ARGS ARG10 diffRun.diff_id=$diffRun.diff_id +ARGS ARG1 rawImfile.exp_id=$lapExp.exp_id +ARGS ARG2 chipRun.chip_id=$lapExp.chip_id +ARGS ARG3 chipRun.chip_id=$lapExp.pair_id +ARGS ARG4 camRun.cam_id=$camRun1.cam_id +ARGS ARG5 camRun.cam_id=$camRun2.cam_id +ARGS ARG6 fakeRun.fake_id=$fakeRun1.fake_id +ARGS ARG7 fakeRun.fake_id=$fakeRun2.fake_id +ARGS ARG8 warpRun.warp_id=$warpRun1.warp_id +ARGS ARG9 warpRun.warp_id=$warpRun2.warp_id +ARGS ARG10 diffRun.diff_id=$diffRun.diff_id -# field size format name show link to extras -FIELD lapRun.seq_id, 6, %d, seq_id -FIELD lapExp.lap_id, 6, %d, lap_id -FIELD lapExp.data_state, 6, %s, State (lapExp) -FIELD lapExp.exp_id, 6, %d, Exp ID, value, rawImfile.php, ARG1 -FIELD lapExp.chip_id, 6, %d, Chip1 (lapExp.chip_id), value, chipStageExp.php, ARG2 -FIELD chipRun1.state AS ch1state, 6, %s, Chip1 State -FIELD camRun1.cam_id AS cam_id_1, 6, %d, Cam1 ID, value, camStageExp.php, ARG4 -FIELD camRun1.state AS ca1state, 6, %s, Cam1 State -FIELD fakeRun1.fake_id AS fake_id_1, 6, %d, Fake1 ID, value, fakeStageExp.php, ARG6 -FIELD fakeRun1.state AS fa1state, 6, %s, Fake1 State -FIELD warpRun1.warp_id AS warp_id_1, 6, %d, Warp1 ID, value, warpStageExp.php, ARG8 -FIELD warpRun1.state AS wa1state, 6, %s, Warp1 State -FIELD lapExp.pair_id, 6, %d, Chip2 ID (lapExp.chip_id), value, chipStageExp.php, ARG3 -FIELD chipRun2.state AS ch2state, 6, %s, Chip2 State -FIELD camRun2.cam_id AS cam_id_2, 6, %d, Cam2 ID, value, camStageExp.php, ARG5 -FIELD camRun2.state AS ca2state, 6, %s, Cam2 State -FIELD fakeRun2.fake_id AS fake_id_2, 6, %d, Fake2 ID, value, fakeStageExp.php, ARG7 -FIELD fakeRun2.state AS fa2state, 6, %s, Fake2 State -FIELD warpRun2.warp_id AS warp_id_2, 6, %d, Warp2 ID, value, warpStageExp.php, ARG9 -FIELD warpRun2.state AS wa2state, 6, %s, Warp2 State -FIELD diffRun.diff_id, 6, %d, Diff ID, value, diffStageExp.php, ARG10 -FIELD diffRun.state AS diState, 6, %s, Diff State -FIELD lapRun.tess_id, 8, %s, tess_id -FIELD lapRun.projection_cell, 10, %s, ProjCell -FIELD lapRun.filter, 7, %s, Filter -FIELD lapRun.state, 15, %s, State (lapRun) -FIELD lapRun.label, 15, %s, Label -FIELD lapRun.dist_group, 15, %s, Dist Group -FIELD lapRun.registered, 19, %T, Registered -FIELD lapRun.fault, 3, %d, fault -FIELD lapRun.quick_sass_id, 6, %d, quick_sass_id -FIELD lapRun.final_sass_id, 6, %d, final_sass_id -FIELD lapExp.private, 2, %d, Private -FIELD lapExp.pairwise, 2, %d, Pairwise -FIELD lapExp.active, 2, %d, Active -FIELD rawExp.dateobs, 19, %T, Date/Time -FIELD rawExp.object, 8, %s, Object -FIELD rawExp.comment, 80, %s, Comment +# field size format name show link to extras +FIELD lapRun.seq_id, 6, %d, seq_id +FIELD lapExp.lap_id, 6, %d, lap_id +FIELD lapExp.data_state, 6, %s, State (lapExp) +FIELD lapExp.exp_id, 6, %d, Exp ID, value, rawImfile.php, ARG1 +FIELD lapExp.chip_id, 6, %d, Chip1 (lapExp.chip_id), value, chipStageExp.php, ARG2 +FIELD chipRun1.state AS ch1state, 6, %s, Chip1 State +FIELD camRun1.cam_id AS cam_id_1, 6, %d, Cam1 ID, value, camStageExp.php, ARG4 +FIELD camRun1.state AS ca1state, 6, %s, Cam1 State +FIELD fakeRun1.fake_id AS fake_id_1, 6, %d, Fake1 ID, value, fakeStageExp.php, ARG6 +FIELD fakeRun1.state AS fa1state, 6, %s, Fake1 State +FIELD warpRun1.warp_id AS warp_id_1, 6, %d, Warp1 ID, value, warpStageExp.php, ARG8 +FIELD warpRun1.state AS wa1state, 6, %s, Warp1 State +FIELD lapExp.pair_id, 6, %d, Chip2 ID (lapExp.chip_id), value, chipStageExp.php, ARG3 +FIELD chipRun2.state AS ch2state, 6, %s, Chip2 State +FIELD camRun2.cam_id AS cam_id_2, 6, %d, Cam2 ID, value, camStageExp.php, ARG5 +FIELD camRun2.state AS ca2state, 6, %s, Cam2 State +FIELD fakeRun2.fake_id AS fake_id_2, 6, %d, Fake2 ID, value, fakeStageExp.php, ARG7 +FIELD fakeRun2.state AS fa2state, 6, %s, Fake2 State +FIELD warpRun2.warp_id AS warp_id_2, 6, %d, Warp2 ID, value, warpStageExp.php, ARG9 +FIELD warpRun2.state AS wa2state, 6, %s, Warp2 State +FIELD diffRun.diff_id, 6, %d, Diff ID, value, diffStageExp.php, ARG10 +FIELD diffRun.state AS diState, 6, %s, Diff State +FIELD lapRun.tess_id, 8, %s, tess_id +FIELD lapRun.projection_cell, 10, %s, ProjCell +FIELD diffInputSkyfile.skycell_id, 10, %s, Skycell ID +FIELD lapRun.filter, 7, %s, Filter +FIELD lapRun.state, 15, %s, State (lapRun) +FIELD lapRun.label, 15, %s, Label +FIELD lapRun.dist_group, 15, %s, Dist Group +FIELD lapRun.registered, 19, %T, Registered +FIELD lapRun.fault, 3, %d, fault +FIELD lapRun.quick_sass_id, 6, %d, quick_sass_id +FIELD lapRun.final_sass_id, 6, %d, final_sass_id +FIELD lapExp.private, 2, %d, Private +FIELD lapExp.pairwise, 2, %d, Pairwise +FIELD lapExp.active, 2, %d, Active +FIELD rawExp.dateobs, 19, %T, Date/Time +FIELD rawExp.object, 8, %s, Object +FIELD rawExp.comment, 80, %s, Comment # MODE summary Index: branches/meh_branches/ppstack_test/ippMonitor/def/mopsStatus.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/mopsStatus.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/mopsStatus.d (revision 33415) @@ -5,5 +5,5 @@ #WHERE rawExp.obs_mode!='ENGINEERING' -UNRESTRICTED WHERE dateobs>=DATE(NOW()) AND rawExp.obs_mode!='ENGINEERING' +UNRESTRICTED WHERE dateobs>=DATE(NOW()) AND rawExp.obs_mode!='ENGINEERING' AND rawExp.obs_mode!='STS' AND rawExp.obs_mode!='M31' AND rawExp.obs_mode!='CNP' # OP OP1 ($ddiff_id > 0 ? $ddiff_id : $tdiff_id) Index: branches/meh_branches/ppstack_test/ippMonitor/def/stackProcessedSkyfile_Images.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/stackProcessedSkyfile_Images.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/stackProcessedSkyfile_Images.d (revision 33415) @@ -18,9 +18,10 @@ # field size format name show link to extras -FIELD stackRun.stack_id, 5, %s, Stack ID +FIELD stackRun.stack_id, 5, %d, Stack ID FIELD stackRun.skycell_id, 5, %s, Skycell ID FIELD stackRun.label, 5, %s, Label FIELD stackRun.data_group, 5, %s, data grp FIELD stackRun.dist_group, 5, %s, dist grp +FIELD stackSumSkyfile.quality, 5, %d, quality FIELD *, 8, %s, image, image=JPEG2, stackProcessedSkyfile_Images.php, ARG7 FIELD stackRun.filter, 5, %s, Filter Index: branches/meh_branches/ppstack_test/ippMonitor/def/warpProcessedSkyfiles.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/warpProcessedSkyfiles.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/warpProcessedSkyfiles.d (revision 33415) @@ -1,13 +1,8 @@ -TABLE warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp +TABLE rawExp JOIN chipRun USING(exp_id) JOIN camRun USING(chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) JOIN warpSkyfile USING(warp_id) TITLE Warp Processed Skyfiles FILE warpProcessedSkyfiles.php MENU ipp.science.dat -WHERE warpSkyfile.warp_id = warpRun.warp_id -WHERE warpRun.fake_id = fakeRun.fake_id -WHERE fakeRun.cam_id = camRun.cam_id -WHERE camRun.chip_id = chipRun.chip_id -WHERE chipRun.exp_id = rawExp.exp_id -WHERE warpSkyfile.fault = 0 +UNRESTRICTED WHERE 0=1 ARGS ARG1 rawImfile.exp_id=$rawExp.exp_id Index: branches/meh_branches/ppstack_test/ippMonitor/def/warpProcessedSkyfiles_Images.d =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/def/warpProcessedSkyfiles_Images.d (revision 33415) @@ -1,13 +1,15 @@ -TABLE warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp +TABLE rawExp JOIN chipRun USING(exp_id) JOIN camRun USING(chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) JOIN warpSkyfile USING(warp_id) TITLE Warp Processed Skyfiles FILE warpProcessedSkyfiles.php MENU ipp.science.dat -WHERE warpSkyfile.warp_id = warpRun.warp_id -WHERE warpRun.fake_id = fakeRun.fake_id -WHERE fakeRun.cam_id = camRun.cam_id -WHERE camRun.chip_id = chipRun.chip_id -WHERE chipRun.exp_id = rawExp.exp_id -WHERE warpSkyfile.fault = 0 +UNRESTRICTED WHERE 0=1 + +# WHERE warpSkyfile.warp_id = warpRun.warp_id +# WHERE warpRun.fake_id = fakeRun.fake_id +# WHERE fakeRun.cam_id = camRun.cam_id +# WHERE camRun.chip_id = chipRun.chip_id +# WHERE chipRun.exp_id = rawExp.exp_id +# WHERE warpSkyfile.fault = 0 # define image names to be used below Index: branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_getplot.php =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_getplot.php (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_getplot.php (revision 33415) @@ -18,5 +18,5 @@ $plottype = $_GET[plottype]; -$path = "/data/ipp004.0/czartool_plots/"; # TODO horrible hard-coded path - read from config +$path = "/data/ipp009.0/czartool_plots"; # TODO horrible hard-coded path - read from config if ($label == "all") $label = "all_".$mode."_labels"; Index: branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_ipptopsps.php =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_ipptopsps.php (revision 33415) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_ipptopsps.php (revision 33415) @@ -0,0 +1,405 @@ + 0) $selectedLabel = $labels[0]; + else $selectedLabel = "no labels"; +} +if ($selectedStage == "") { $selectedStage = "all_stages"; } +if ($plotType == "") { $plotType = "linear"; } + +// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later +if ($selectedRevertStage && $selectedRevertMode) turnRevertsOnOff($czardb, $selectedRevertStage, $selectedRevertMode); + +// tell selected server to stop or run +if ($selectedServer && $serverCmd) serverStopRun($czardb, $selectedServer, $serverCmd); + +// tell all servers to stop or run +if ($allServerCmd) { + + foreach ($servers as &$server) serverStopRun($czardb, $server, $allServerCmd); +} + + +$debug = 0; +$table = "\n"; + + +if ($debug) { + echo "prog returned $status, and output:
"; + for ($i = 0; $i < count($distLabels); $i++) { + echo "$distLabels[$i]
"; + } +} + +$isUpdate = ($selectedMode == "update"); + +// set up the form +echo "\n"; + +echo "\n"; +echo "\n"; +echo "\n"; +echo "\n"; + +echo "
"; + +echo ""; +echo ""; +echo ""; + +echo ""; +echo ""; +echo "\n"; +echo ""; +echo ""; + +echo "
"; +echo "

Czartool

"; +echo "

IPP to PSPS Monitoring

"; +echo "
"; + # time series plot +echo "
"; + echo "
"; + # rate time series plot +echo "
"; + echo "
"; +echo "		"; +echo ""; + +#echo "
"; + +// status table at top +$plotTypeLink = ($plotType == "linear") ? "log" : "linear"; +$link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; +echo "
"; +echo $table; +createTableTitle("Status and links", 3); +echo "
Time of last update$lastUpdateTime
Documentationhere
\n"; + +$plotTypeLink = ($plotType == "linear") ? "log" : "linear"; +$link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; + + + +echo "
"; + + createLabelsTable($pass, + $proj, + $czardb, + $selectedMode, + $labels, + $distLabels, + $pubLabels, + $stages, + "new", + $selectedLabel, + $selectedStage, + $plotType); + +echo "
"; + +echo ""; + +menu_end(); + +########################################################################### +# +# Gets labels in use over the last 24 hours +# +########################################################################### +function getLabels($db) { + + # order by descending priority as set in gpc1 database + $sql = "SELECT DISTINCT label FROM processed WHERE timestamp >= now() - INTERVAL 1 DAY ORDER BY label"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + while ($qry->fetchInto($row)) { + + $labels[] = $row[0]; + } + + return $labels; +} + +########################################################################### +# +# Creates table for all labels showing all stages for given 'state' +# +########################################################################### +function createLabelsTable( + $pass, + $proj, + $db, + $selectedMode, + $labels, + $distLabels, + $pubLabels, + $stages, + $selectedState, + $selectedLabel, $selectedStage, $plotType) { + + $isUpdate = ($selectedMode == "update"); + global $headerColor; + global $columnHeaderColor; + + // setup table + global $table; + echo $table; + createTableTitle("Label status", 13); + + echo "\n"; + $link = "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=".$selectedMode + ."&label=".$selectedLabel + ."&stage=all_stages" + ."&plottype=".$plotType; + createFormattedTableCell(($selectedStage != "all_stages"), $link, "All stages", 0, "$columnHeaderColor"); + + #echo "Label (in order of priority)"; + foreach ($stages as &$stage) { + + if ($stage == $selectedStage) $showLink = 0; + else $showLink = 1; + $link = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$selectedLabel + ."&stage=".$stage + ."&plottype=".$plotType; + + createFormattedTableCell($showLink, $link, $stage, 0, "$columnHeaderColor"); +# createTableColumnHeader("Dates"); + } + + // setup default link + $defaultlink = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$selectedLabel + ."&stage=".$selectedStage + ."&plottype=".$plotType; + + $searchState = "new"; + if ($selectedMode == "update") $searchState = "update"; + + // write rows + foreach ($labels as &$thisLabel) { + + + $distributing = false; + $publishing = false; + + // show distribution/publishing status + echo "\n"; + + // create link to label summary page for each label + if ($thisLabel == $selectedLabel) $showLink = 0; + else $showLink = 1; + $link = + "czartool_ipptopsps.php?pass=".$pass + ."&proj=".$proj + ."&mode=" .$selectedMode + ."&label=".$thisLabel + ."&stage=".$selectedStage + ."&plottype=".$plotType; + createFormattedTableCell($showLink, $link, $thisLabel, 0, null); + + $str = ""; + $anyFaults = false; + + $link = $defaultlink; + getStateAndFaults($db, $thisLabel, $selectedState, "processed", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "loaded_to_datastore", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "loaded_to_ODM", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "merge_worthy", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_datastore", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_dxlayer", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "merged", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + getStateAndFaults($db, $thisLabel, $selectedState, "deleted_local", $str, $anyFaults); + createFormattedTableCell(0, $link, $str, $anyFaults, 0); + + echo "\n"; + } + + echo "\n"; + $link = "czartool_ipptopsps.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=all&stage=".$selectedStage."&plottype=".$plotType; + + echo "\n"; + if ($selectedLabel == "all") $showLink = 0; + else $showLink = 1; + $link = "czartool_ipptopsps.php?pass=".$pass."&proj=".$proj."&mode=".$selectedMode."&label=all&stage=".$selectedStage."&plottype=".$plotType; + + echo "\n"; + echo "\n"; +} + + +########################################################################### +# +# Creates a table title +# +########################################################################### +function createTableTitle($str, $colSpan) { + + global $headerColor; + echo "

$str

"; +} + +########################################################################### +# +# Creates a table header cell +# +########################################################################### +function createTableColumnHeader($str) { + + global $columnHeaderColor; + echo "

$str

"; +} + +########################################################################### +# +# Creates table cell with a red background if an error +# +########################################################################### +function createFormattedTableCell($showLink, $link, $str, $error, $color) { + + $fontColor = $error ? "black" : "blue"; + + if ($error) echo ""; + else if ($color) echo ""; + else echo ""; + if ($showLink) echo "$str"; + else if ($str) echo "$str"; + else echo " "; +} + + +########################################################################### +# +# Returns state and fault-count (if new) as a string +# +########################################################################### +function getStateAndFaults($db, $label, $state, $stage, &$str, &$anyFaults) { + + $anyFaults = false; + $sql = "SELECT pending, faults FROM $stage WHERE label LIKE '$label' ORDER BY timestamp DESC LIMIT 1"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + $pending = $row[0]; + $faults = $row[1]; + + if ($pending == 0) + $str = ""; + else + $str = "$pending"; + + if ($state == "new") { + + if ($faults > 0) { + + $str = $str."(".$faults.")"; + $anyFaults = true; + } + } +} + +########################################################################### +# +# Returns time of last czarDb update +# +########################################################################### +function getLastUpdateTime($db) { + + $anyFaults = false; + $sql = "SELECT timestamp FROM processed ORDER BY timestamp DESC LIMIT 1"; + if ($debug) {echo "$sql
";} + + $qry = $db->query($sql); + if (dberror($qry)) {echo "error with $sql
\n";} + $qry->fetchInto($row); + + return $row[0]; +} + + +?> + Index: branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_labels.php =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_labels.php (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/czartool_labels.php (revision 33415) @@ -27,6 +27,17 @@ menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']); -$states=array("full","new","drop","wait"); -$stages=array("burntool", "chip","cam","fake","warp","stack","diff","magic","magicDS","dist", "pub"); +$stages=array( + "burntool", + "chip", + "cam", + "fake", + "warp", + "stack", + "diff", + "magic", + "magicDS", + "dist", + "pub"); + $servers=array( "addstar", @@ -41,4 +52,5 @@ "replication", "stack", + "deepstack", "stdscience", "summitcopy"); @@ -149,5 +161,5 @@ echo "
"; echo $table; -#"\n"; + createTableTitle("Status and links", 3); echo "\n"; @@ -157,7 +169,7 @@ echo ""; echo ""; -echo ""; -echo ""; -echo ""; +echo ""; +echo ""; +echo ""; echo ""; echo ""; @@ -165,5 +177,4 @@ $plotTypeLink = ($plotType == "linear") ? "log" : "linear"; $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&mode=" . $selectedMode . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink"; -echo ""; @@ -180,5 +191,4 @@ $pubLabels, $stages, - $states, "new", $selectedLabel, @@ -201,5 +211,5 @@ echo ""; createTableColumnHeader("Database"); - createTableColumnHeader("Status"); + createTableColumnHeader("Status (sec behind master)"); showReplicationsStatus($REPL_HOST_GPC1, $REPL_USER_GPC1, $REPL_PASSWORD_GPC1, $REPL_DBNAME_GPC1); showReplicationsStatus($REPL_HOST_NEBULOUS, $REPL_USER_NEBULOUS, $REPL_PASSWORD_NEBULOUS, $REPL_DBNAME_NEBULOUS); @@ -349,5 +359,4 @@ $pubLabels, $stages, - $states, $selectedState, $selectedLabel, $selectedStage, $plotType) { @@ -483,5 +492,5 @@ createFormattedTableCell(($selectedLabel != "all"), $link, "All labels", 0, null); - createFormattedTableCell(0, null, "Reverts:", 0, "$columnHeaderColor"); + createFormattedTableCell(0, null, "Reverting?", 0, "$columnHeaderColor"); foreach ($stages as &$stage) { @@ -489,18 +498,21 @@ if ($stage == "burntool") continue; $reverting = getRevertStatus($db, $stage); - $link = - "czartool_labels.php?pass=".$pass - ."&proj=".$proj - ."&mode=" .$selectedMode - ."&label=".$selectedLabel - ."&stage=".$selectedStage - ."&plottype=".$plotType - ."&revertstage=".$stage - ."&revertmode="; - - if(!$reverting) {$label = "Start";$link = $link . "on";} - if($reverting) {$label = "Stop";$link = $link . "off";} + #$link = + # "czartool_labels.php?pass=".$pass + # ."&proj=".$proj + # ."&mode=" .$selectedMode + # ."&label=".$selectedLabel + # ."&stage=".$selectedStage + # ."&plottype=".$plotType + # ."&revertstage=".$stage + # ."&revertmode="; +# + #if(!$reverting) {$label = "Start";$link = $link . "on";} + #if($reverting) {$label = "Stop";$link = $link . "off";} + if(!$reverting) {$label = "no";} + if($reverting) {$label = "yes";} unset($reverting); - createFormattedTableCell(1, $link, $label, 0, "$columnHeaderColor"); + #createFormattedTableCell(1, $link, $label, 0, "$columnHeaderColor"); + createFormattedTableCell(0, null, $label, 0, "$columnHeaderColor"); #echo ""; @@ -904,5 +916,5 @@ //die("MySQL DB connection error: Check the configuration for $replDatabaseName"); //die(""); - return; + return -1; } @@ -911,5 +923,6 @@ # Have a look for Last_Errno in http://dev.mysql.com/doc/refman/5.0/en/show-slave-status.html $errorStatusInMySql = $row[18]; - $replStatus = ($errorStatusInMySql==0?"OK":"PROBLEM"); + $sec_behind = $row[32]; + $replStatus = ($errorStatusInMySql==0?"OK ($sec_behind)":"PROBLEM"); echo ""; if ($errorStatusInMySql!=0) { Index: branches/meh_branches/ppstack_test/ippMonitor/raw/ipp.imfiles.dat =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/ipp.imfiles.dat (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/ipp.imfiles.dat (revision 33415) @@ -21,5 +21,6 @@ menutop | menutop | link | simple plot - chip (to remove?) | simplePlot.php menutop | menutop | link | simple plot - cam (to remove?) | simplePlotcam.php -menutop | menutop | link | czartool | czartool_labels.php +menutop | menutop | link | czartool - standard | czartool_labels.php +menutop | menutop | link | czartool - ipptopsps | czartool_ipptopsps.php menutop | menutop | link | mask stats | maskStats.php Index: branches/meh_branches/ppstack_test/ippMonitor/raw/nightSummary.php =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/nightSummary.php (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/nightSummary.php (revision 33415) @@ -121,4 +121,6 @@ table_section($db, $WHERE, "magicRun"); table_section($db, $WHERE, "destreakRun"); +table_section($db, $WHERE, "chipBackgroundRun"); +table_section($db, $WHERE, "warpBackgroundRun"); table_section($db, $WHERE, "distRun"); // table_section($db, $WHERE, "publishRun"); Index: branches/meh_branches/ppstack_test/ippMonitor/raw/warpProcessedExp.php =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/warpProcessedExp.php (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/warpProcessedExp.php (revision 33415) @@ -117,5 +117,5 @@ // query the database if (basic == "basic") { - $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart"; + $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) JOIN warpSummary USING(warp_id) $WHERE LIMIT $dTABLE OFFSET $rowStart"; } Index: branches/meh_branches/ppstack_test/ippMonitor/raw/warpProcessedExp_Images.php =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/raw/warpProcessedExp_Images.php (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/raw/warpProcessedExp_Images.php (revision 33415) @@ -77,12 +77,10 @@ if ($restricted == 0) { - if ("none" != "none") { - $WHERE = "$WHERE none"; - } + $WHERE = "WHERE 0=1"; } // get the result table count if (basic == "basic") { - $sql = "SELECT count(*) FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE"; + $sql = "SELECT count(*) FROM rawExp JOIN chipRun USING(exp_id) JOIN camRun USING (chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE"; } @@ -111,5 +109,5 @@ // query the database if (basic == "basic") { - $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart"; + $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage'),warpRun.tess_id FROM rawExp JOIN chipRun USING(exp_id) JOIN camRun USING (chip_id) JOIN fakeRun USING(cam_id) JOIN warpRun USING(fake_id) LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart"; } @@ -175,4 +173,5 @@ $buttonLink = button_restrict_string ('warpSummary.path_base', $buttonLink); $buttonLink = button_restrict_string ('warpSummary.projection_cell', $buttonLink); +$buttonLink = button_restrict_string ('warpRun.tess_id', $buttonLink); navigate_buttons ($rowStart, $rowLast, $dTABLE, $rowTotal, $buttonLink, $ID, 'warpProcessedExp_Images.php'); @@ -210,4 +209,5 @@ write_header_cell ("list", "stdev "); write_header_cell ("list", "Comment"); +write_header_cell ("list", "TessId"); echo "\n"; echo "\n"; @@ -236,4 +236,5 @@ write_sort_cell ("list", "rawExp.bg_stdev", $buttonLink, $ID, 'warpProcessedExp_Images.php'); write_sort_cell ("list", "rawExp.comment", $buttonLink, $ID, 'warpProcessedExp_Images.php'); +write_sort_cell ("list", "warpRun.tess_id", $buttonLink, $ID, 'warpProcessedExp_Images.php'); echo "\n"; // echo "\n"; @@ -268,4 +269,5 @@ write_query_row ('rawExp.bg_stdev', 5, 'min'); write_query_row ('rawExp.comment', 65, 'string'); +write_query_row ('warpRun.tess_id', 65, 'string'); echo "\n"; echo "\n"; @@ -292,4 +294,5 @@ write_query_row ('rawExp.bg_stdev', 5, 'max'); echo "\n"; +echo "\n"; echo "\n"; // echo "\n"; @@ -312,13 +315,14 @@ //Identify current warpId $warpId = $row[5]; -/* echo "\n"; */ -/* echo "\n"; */ + $tessId = $row[24]; + /* echo "\n"; */ + /* echo "\n"; */ if (!isSet($warpIdCollection[$warpId])) { -/* echo ""; */ + /* echo ""; */ $warpIdCollection[$warpId] = array(); $skyCellValue = preg_replace("|skycell[.]|","",$row[23]); $warpIdCollection[$warpId]["12"] = $skyCellValue; -/* echo ""; */ + /* echo ""; */ echo "\n"; @@ -344,10 +348,10 @@ echo "
Postage stamp server statushere
Documentationhere
Cluster loadhere
Who's using the cluster?here
IPP metricshere
Cluster loadhere cpu_report
Who's using the cluster?here
IPP metricshere
Czar log pageshere
Exposure summaryhere
Plot typeChange to $plotTypeLink
$label
$replDatabaseName$replStatus
≤      
Number of elements in warpIdCollection: ".count($warpIdCollection)."
(warpId=$warpId)
Number of elements in warpIdCollection: ".count($warpIdCollection)."
(warpId=$warpId)
$warpId not in warpIdCollection
$warpId not in warpIdCollection
Skycell is [$skyCellValue]
Skycell is [$skyCellValue]
\n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; echo " \n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; @@ -364,10 +368,10 @@ echo "
$noImageValue$noImageValue$noImageValue$noImageValue
$noImageValue$noImageValue$noImageValue\n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; echo " \n"; echo "\n"; -/* echo " \n"; */ + /* echo " \n"; */ echo " \n"; echo " \n"; @@ -376,9 +380,18 @@ echo "\n"; - $cellContent = " "; - echo ""; - $imageNameValue = ($row[23]=="noimage" ? - "No image" : $row[23]); - echo ""; + if ($tessId != "RINGS.V3") { + $cellContent = " "; + echo ""; + $imageNameValue = ($row[23]=="noimage" ? + "No image" : $row[23]); + echo ""; + } else { + //The cell to update is the upper right one for new tessellation + $cellContent = " "; + echo ""; + $imageNameValue = ($row[23]=="noimage" ? + "No image" : $row[23]); + echo ""; + } //resume with "regular" columns @@ -399,4 +412,5 @@ write_table_cell ($class, '%.2f', "", $row[20]); write_table_cell ($class, '%s', "", $row[21]); + write_table_cell ($class, '%s', "", $row[24]); echo "\n"; @@ -415,36 +429,77 @@ $cellContent = " "; - if ($skyCellValue == $lrValue + 1) { - $warpIdCollection[$warpId]["11"] = $skyCellValue; - echo ""; - echo ""; - } else { - if ($skyCellValue == $lrValue + 2) { - $warpIdCollection[$warpId]["10"] = $skyCellValue; - echo ""; - echo ""; + if ($tessId != "RINGS.V3") { + if ($skyCellValue == $lrValue + 1) { + $warpIdCollection[$warpId]["11"] = $skyCellValue; + echo ""; + echo ""; } else { - if (!isSet($warpIdCollection[$warpId]["02"])) { - /* echo ""; */ - $warpIdCollection[$warpId]["02"] = $skyCellValue; - echo ""; - echo ""; - } else { - $urValue = $warpIdCollection[$warpId]["02"]; - if ($urValue + 1 == $skyCellValue) { - echo ""; - echo ""; + if ($skyCellValue == $lrValue + 2) { + $warpIdCollection[$warpId]["10"] = $skyCellValue; + echo ""; + echo ""; + } else { + if (!isSet($warpIdCollection[$warpId]["02"])) { + /* echo ""; */ + $warpIdCollection[$warpId]["02"] = $skyCellValue; + echo ""; + echo ""; } else { - if ($urValue + 2 == $skyCellValue) { - echo ""; - echo ""; + $urValue = $warpIdCollection[$warpId]["02"]; + if ($urValue + 1 == $skyCellValue) { + echo ""; + echo ""; } else { - $expected = $urValue + 1; - $actual = $skyCellValue; - echo ""; - echo ""; - } - } + if ($urValue + 2 == $skyCellValue) { + echo ""; + echo ""; + } else { + $expected = $urValue + 1; + $actual = $skyCellValue; + echo ""; + echo ""; + } + } + } } + } + } else { /* New tessellation */ + + /* echo "

$lrValue - $skyCellValue

"; */ + + if ($skyCellValue == $lrValue + 1) { + $warpIdCollection[$warpId]["11"] = $skyCellValue; + echo ""; + echo ""; + } else { + if ($skyCellValue == $lrValue + 2) { + $warpIdCollection[$warpId]["10"] = $skyCellValue; + echo ""; + echo ""; + } else { + if (!isSet($warpIdCollection[$warpId]["02"])) { + /* echo ""; */ + $warpIdCollection[$warpId]["02"] = $skyCellValue; + echo ""; + echo ""; + } else { + $urValue = $warpIdCollection[$warpId]["02"]; + if ($urValue + 1 == $skyCellValue) { + echo ""; + echo ""; + } else { + if ($urValue + 2 == $skyCellValue) { + echo ""; + echo ""; + } else { + $expected = $urValue + 1; + $actual = $skyCellValue; + echo ""; + echo ""; + } + } + } + } + } } Index: branches/meh_branches/ppstack_test/ippMonitor/scripts/skyplot.dvo =================================================================== --- branches/meh_branches/ppstack_test/ippMonitor/scripts/skyplot.dvo (revision 32161) +++ branches/meh_branches/ppstack_test/ippMonitor/scripts/skyplot.dvo (revision 33415) @@ -8,4 +8,5 @@ black gold + orange end @@ -17,4 +18,5 @@ 7 3 + 0 end @@ -26,4 +28,5 @@ 0.9 1.3 + 0.5 end Index: branches/meh_branches/ppstack_test/ippScripts/Build.PL =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/Build.PL (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/Build.PL (revision 33415) @@ -65,5 +65,9 @@ scripts/minidvodb_createdb.pl scripts/minidvodb_merge.pl + scripts/minidvodb_premerge.pl scripts/minidvodb_copy.pl + scripts/mergedvodb_copy.pl + scripts/mergedvodb_merge.pl + scripts/mergedvodb_queue.pl scripts/ippdb.pl scripts/ipp_cleanup.pl @@ -71,4 +75,5 @@ scripts/ipp_inject_expname.pl scripts/ipp_serial_inject.pl + scripts/ipp_serial_inject_into_nebulous.pl scripts/ipp_serial_inject_split.pl scripts/ipp_serial_inject_mosaic.pl @@ -94,4 +99,5 @@ scripts/dist_bundle.pl scripts/dist_advancerun.pl + scripts/dist_fix_file.pl scripts/dist_make_fileset.pl scripts/dist_defineruns.pl @@ -120,4 +126,5 @@ scripts/videophot_process.pl scripts/listvideocells.pl + scripts/skycalibration.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: branches/meh_branches/ppstack_test/ippScripts/scripts/addstar_run.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/addstar_run.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/addstar_run.pl (revision 33415) @@ -30,4 +30,5 @@ my $missing_tools; my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1); my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); @@ -37,5 +38,6 @@ } my $minidvodb_path; -my ( $add_id, $camera, $stage, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, +my $stage_extra1 ; +my ( $add_id, $camera, $stage, $stage_id, $multi_num, $outroot, $stageroot, $dbname, $reduction, $dvodb, $minidvodb, $minidvodb_name, $minidvodb_group, $image_only, $verbose, $no_update, $no_op, $redirect, $save_temps); GetOptions( @@ -43,5 +45,6 @@ 'camera|c=s' => \$camera, # Camera 'stage|s=s' => \$stage, # Camera - + 'stage_id|w=s' => \$stage_id, + 'multi_num|w=s' => \$multi_num, # the number for a staticskymulti (for finding cmf) 'dbname|d=s' => \$dbname, # Database name 'outroot|w=s' => \$outroot, # output file base name @@ -71,4 +74,8 @@ defined $dvodb and defined $camera; +if ($stage =~ /cam/ && !defined $stage_id) { + my_die("cam stage requires -stage_id", $add_id, 3); + +} if ($minidvodb && !defined($minidvodb_group)) { my_die( "missing minidvodb_group", $add_id, 3 ); @@ -108,12 +115,61 @@ # it was PSASTRO.OUTPUT -my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + +my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT", $stageroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + +if ($stage =~ /cam/) { + # if it is cam stage we need to be careful when grabbing the filename. + # This breaks down into a few steps: + + #get info about the cam_id + my $magicked; + { + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$camtool -processedexp -cam_id " . $stage_id; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get info on camRun", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + &my_die( "empty camRun", $add_id, $PS_EXIT_SYS_ERROR); + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mparsed = $comp->{magicked}; + + if (!defined($mparsed)) { + &my_die("Unable to parse magicked", $add_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found a value for magicked:$mparsed\n"; + $magicked = $mparsed; + } + + #is this cam_id magicked or not? + if ($magicked) { + $stage_extra1 = $magicked; + $fpaObjects = $ipprc->destreaked_filename("$fpaObjects") or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); + print "cam_id is magicked, using $fpaObjects for the cam smf\n"; + } else { + print "cam_id is NOT magicked, using $fpaObjects for the cam smf\n"; + } + } +} if (($stage =~/staticsky/) || ($stage =~/stack/)) { $fpaObjects =~ s/smf$/cmf/; - -} +} +if ($stage =~/staticsky_multi/) { + $fpaObjects =~ s/smf$/cmf/; #this should do nothing because it was already modded above + &my_die( "can't find the filter_num for staticsky_multi, giving up.", $add_id, $PS_EXIT_SYS_ERROR) unless (defined $multi_num); + my $nice_num = sprintf ("%03d", $multi_num); + $fpaObjects =~ s/cmf$/$nice_num.cmf/; #this make it look for .001.cmf, etc +} + my $traceDest = $ipprc->filename("TRACE.EXP", $outroot) or &my_die("Missing entry from camera config", $add_id, $PS_EXIT_CONFIG_ERROR); - + # convert supplied DVO database name to UNIX filename my $dvodbReal; @@ -122,84 +178,69 @@ $dvodbReal = $ipprc->convert_filename_absolute( $dvodbReal ) or &my_die("can't get path for dvodb", $add_id, $PS_EXIT_CONFIG_ERROR); } - - - - my $dtime_addstar = 0; - if (defined $dvodbReal) { - if ($minidvodb) { - - #if (!defined($minidvodb_name)) { - #take the active one, if it's not defined on the command line - #reverts would have this already set, for example. - my $command = "addtool -listminidvodbrun "; - $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; - $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; - $command .= " -state 'active' -limit 1"; - $command .= " -dbname $dbname" if defined $dbname; - print $command; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); - - my $components = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); - my $comp = $$components[0]; - $minidvodb_path = $comp->{minidvodb_path}; - $minidvodb_name = $comp->{minidvodb_name}; - - if (!defined($minidvodb_path)) { - &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); - } - if (!defined($minidvodb_name)) { - &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); - } - #} - # tack on the minidvodb part to the db. -# $dvodbReal = $dvodbReal . '/' . $minidvodb_name . '/'; -#we don't need this now that I fixed the paths - - } - else { - - $minidvodb_path = $dvodbReal; - } - -unless ($no_op) { + if ($minidvodb) { + my $command = "addtool -listminidvodbrun "; + $command .= " -minidvodb_group $minidvodb_group" if defined $minidvodb_group; + $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + $command .= " -state 'active' -limit 1"; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get active minidvodb_name", $add_id, $PS_EXIT_SYS_ERROR) unless $success; + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $add_id, $PS_EXIT_PROG_ERROR); + + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $add_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + $minidvodb_path = $comp->{minidvodb_path}; + $minidvodb_name = $comp->{minidvodb_name}; + + if (!defined($minidvodb_path)) { + &my_die("Unable to parse minidvodb_path", $add_id, $PS_EXIT_PROG_ERROR); + } + if (!defined($minidvodb_name)) { + &my_die("Unable to parse minidvodb_name", $add_id, $PS_EXIT_PROG_ERROR); + } + } + else { + $minidvodb_path = $dvodbReal; + } + + unless ($no_op) { print $dvodbReal; - - ## addstar can either save the full set of detections, or just - ## the image metadata, in the dvodb. this is set in the - ## database table addRun - - # addstar requires the user to have a valid .ptolemyrc which - # in turn points at ippconfig/dvo.site - - # get the names for the camera and the real input file - my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar - my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); - - # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) - my $command = "$addstar -update"; # XXX optionally set -update? - $command .= " -D CAMERA $camdir"; - $command .= " -D CATDIR $minidvodb_path"; - $command .= " $realFile"; - $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference + + ## addstar can either save the full set of detections, or just + ## the image metadata, in the dvodb. this is set in the + ## database table addRun + + # addstar requires the user to have a valid .ptolemyrc which + # in turn points at ippconfig/dvo.site + + # get the names for the camera and the real input file + my $camdir = $ipprc->dvo_cameradir(); # Camera directory for addstar + my $realFile = $ipprc->file_resolve($fpaObjects) or &my_die("Unable to resolve $fpaObjects", $add_id, $PS_EXIT_SYS_ERROR); + + # require a defined output dvo database to run addstar (ie, refuse to use the .ptolemyrc default) + my $command = "$addstar -update"; # XXX optionally set -update? + $command .= " -D CAMERA $camdir"; + $command .= " -D CATDIR $minidvodb_path"; + $command .= " $realFile"; + $command .= " -use-name $fpaObjects"; # DVO wants the neb-name as a file reference $command .= " -image" if $image_only; - if ($stage = ~/staticsky/) { + if ($stage =~ /staticsky/) { $command .= " -accept-astrom "; + } #careful here - this matches staticsky and staticsky_multi + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); } - my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform addstar: $error_code", $add_id, $error_code); - } - $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); # MJD of starting script } } @@ -212,4 +253,5 @@ $fpaCommand .= " -dvodb_path $minidvodb_path" if defined $minidvodb_path; $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; +$fpaCommand .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $fpaCommand .= " -dbname $dbname" if defined $dbname; print $fpaCommand; @@ -244,6 +286,8 @@ $command .= " -path_base $outroot" if defined $outroot; $command .= (" -dtime_addstar " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $fpaCommand .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; + # $command .= " -minidvodb_name $minidvodb_name" if defined $minidvodb_name; don't think we want it recorded (not sure) + $command .= " -stage_extra1 $stage_extra1" if defined $stage_extra1; $command .= " -dbname $dbname" if defined $dbname; + print $command; system ($command); } Index: branches/meh_branches/ppstack_test/ippScripts/scripts/background_chip.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/background_chip.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/background_chip.pl (revision 33415) @@ -42,5 +42,5 @@ # Parse the command-line arguments my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose, - $threads, $no_update, $save_temps, $no_op, $redirect, $deburned ); + $threads, $no_update, $save_temps, $no_op, $redirect, $chip_path_base, $magicked ); GetOptions( 'chip_bg_id=s' => \$chip_bg_id, # chipBackgroundRun identifier @@ -51,4 +51,6 @@ 'reduction=s' => \$reduction, # Reduction class 'threads=s' => \$threads, # Number of threads to use + 'chip_path_base=s' => \$chip_path_base, # optional chip_path_base + 'magicked=s' => \$magicked, # magicked status of input 'verbose' => \$verbose, # Print to stdout 'no-update' => \$no_update, # Don't update the database? @@ -88,6 +90,17 @@ # Get inputs my $in_path; # Input path -my $magicked; # Input is magicked? -{ +my $do_stats; +my $dump_config; +if ($chip_path_base and $no_update) { + # we are running outside of a chip_bg_run (perhaps by the postage stamp server) don't dump config + # or do stats. Get path to input from command line. + $in_path = $chip_path_base; + $magicked = 0 if !defined $magicked; + $do_stats = 0; + $dump_config = 0; +} else { + # normal operation. Get input parameters from the database + $do_stats = 1; + $dump_config = 1; my $command = "bgtool -chipinputs -chip_bg_id $chip_bg_id -class_id $class_id"; $command .= " -dbname $dbname" if defined $dbname; @@ -153,11 +166,11 @@ $command .= " -image $in_image"; $command .= " -mask $in_mask"; - $command .= " -stats $out_stats"; + $command .= " -stats $out_stats" if $do_stats; $command .= " -background $in_bg" if $apply_bg; $command .= " -pattern $in_pattern" if $apply_pattern; $command .= " -recipe PPBACKGROUND $recipe_ppBackground"; - $command .= " -recipe PPSTATS CHIPSTATS"; + $command .= " -recipe PPSTATS CHIPSTATS" if $do_stats; $command .= " -dbname $dbname" if defined $dbname; - $command .= " -dumpconfig $out_config"; + $command .= " -dumpconfig $out_config" if $dump_config; $command .= " -tracedest $traceDest -log $logDest"; @@ -171,7 +184,7 @@ # Gather command-line arguments from statistics -my $cmdflags; # Command-line flags to add -my $quality; # Quality flag -{ +my $cmdflags = ""; # Command-line flags to add +my $quality = 0; # Quality flag +if ($do_stats) { &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats); @@ -191,7 +204,7 @@ if (!$quality and !$no_op) { - &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); - &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); - &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config); + &my_die("Couldn't find expected output file: $out_image", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image); + &my_die("Couldn't find expected output file: $out_mask", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask); + &my_die("Couldn't find expected output file: $out_config", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$dump_config or $ipprc->file_exists($out_config); } Index: branches/meh_branches/ppstack_test/ippScripts/scripts/background_warp.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/background_warp.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/background_warp.pl (revision 33415) @@ -38,4 +38,5 @@ my ($warp_bg_id, $skycell_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($astrometry, $imageName, $maskName, $magicked); GetOptions( 'warp_bg_id|i=s' => \$warp_bg_id, # Warp identifier @@ -46,4 +47,8 @@ 'reduction=s' => \$reduction, # Reduction class 'outroot=s' => \$outroot, # Output root name + 'image-list=s' => \$imageName, # list of input image files + 'mask-list=s' => \$maskName, # list of input mask files + 'astrometry=s' => \$astrometry, + 'magicked=s' => \$magicked, 'threads=s' => \$threads, # Number of threads to use for pswarp 'verbose' => \$verbose, # Print to stdout @@ -64,5 +69,20 @@ and defined $outroot; -my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration +if ($imageName) { + # we have been invoked in a special mode where the caller tells us all about the inputs + # This is used by the postage stamp server to make non-background corrected images without + # the existence of a warpBackgroundRun (or chipBackgroundRun) + # Make sure all of the inputs were supplied + if (!(defined $maskName and defined $astrometry and defined $magicked)) { + print STDERR "mask-list, astrometry, and magicked must be supplied if image-list is used\n"; + if ($warp_bg_id) { + my_die( "invalid argument list.", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); + } + exit $PS_EXIT_CONFIG_ERROR; + } +} + +my $ipprc = PS::IPP::Config->new( $camera ) or +my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR ); @@ -97,9 +117,9 @@ # Get list of filenames my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id"; -my ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); -my ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); -my $astrometry; # Astrometry filename -my $magicked; # Magicked status -{ +if (!defined $imageName) { + # go find our inputs + my ($imageFile, $maskFile); + ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX", UNLINK => !$save_temps); + ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX", UNLINK => !$save_temps); my $command = "$bgtool -warpinputs"; $command .= " -warp_bg_id $warp_bg_id"; @@ -132,7 +152,7 @@ $astrometry = $astrom; } -} -close $imageFile; -close $maskFile; + close $imageFile; + close $maskFile; +} &my_die("Can't find input astrometry: $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($astrometry); Index: branches/meh_branches/ppstack_test/ippScripts/scripts/camera_exp.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/camera_exp.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/camera_exp.pl (revision 33415) @@ -88,5 +88,5 @@ if ($redirect) { - $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR ); + $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR ); print "\n\n"; print "Starting script $0 on $host\n\n"; @@ -156,4 +156,10 @@ { # Determine if FWHM is too large to bother continuing. my $maxFWHM = 0; + # XXX: temporary hack if this is a CNP exposure set maxFWHM to a large + # value. + # Before the next tag we will put this into a recipe + if ($outroot =~ /CNP/) { + $maxFWHM = 25; + } else { my $command = "$ppConfigDump -camera $camera -get-key MAX_ALLOWED_FWHM"; my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -168,4 +174,5 @@ $maxFWHM = metadataLookupStr($cameraConfig, 'MAX_ALLOWED_FWHM'); } + } if ($maxFWHM) { my $expFWHM; @@ -403,4 +410,18 @@ &my_die("Couldn't find expected output file: $file", $cam_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $cam_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $cam_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + my $scheme = file_scheme($file); Index: branches/meh_branches/ppstack_test/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/chip_imfile.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/chip_imfile.pl (revision 33415) @@ -34,4 +34,6 @@ my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1); + if ($missing_tools) { warn("Can't find required tools."); @@ -44,6 +46,4 @@ my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose, $no_update, $save_temps, $no_op, $redirect, $magicked, $deburned ); - -my $zaplog = 0; GetOptions( @@ -65,5 +65,4 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, - 'zaplog' => \$zaplog, 'save-temps' => \$save_temps, # Save temporary files? ) or pod2usage( 2 ); @@ -92,15 +91,18 @@ } -my ($logDest, $traceDest); +my ($logRule, $traceDest); if ($run_state eq 'new') { - $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, $zaplog); + $logRule = "LOG.IMFILE"; $traceDest = prepare_output("TRACE.IMFILE", $outroot, $class_id, 1); } else { - $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1); + $logRule = "LOG.IMFILE.UPDATE"; $traceDest = prepare_output("TRACE.IMFILE.UPDATE", $outroot, $class_id, 1); } if ($redirect) { - $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR ); + my $logDest = $ipprc->filename($logRule, $outroot, $class_id); + + $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR ); + print STDOUT "\n\n"; print STDOUT "Starting script $0 on $host\n\n"; @@ -145,9 +147,10 @@ my $outputBin2; my $dump_config = 1; +my $do_stats = 1; +$outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); my $do_binned_images = 1; if ($run_state eq 'new') { # prepare the files that are only created for a new run $configuration = prepare_output("PPIMAGE.CONFIG", $outroot, $class_id, 1); - $outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); } else { $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) @@ -194,5 +197,4 @@ unless ($no_op) { my $command; - my $do_stats; ## get the ppImage recipe for this camera and CHIP reduction @@ -266,27 +268,28 @@ } - ## Read camera config to get the current good burntool state - ## this returns just the values of interest in a tiny mdc file - my $camera_config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD -get-key BURNTOOL.STATE.GOOD.UPDATE"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $camera_config_cmd, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } - - my $camData = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - - my $burntoolStateGood; - my $burntoolStateGoodUpdate; - foreach my $camEntry (@$camData) { - if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") { - $burntoolStateGood = $camEntry->{value}; - } - if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD.UPDATE") { - $burntoolStateGoodUpdate = $camEntry->{value}; - } - } + ## Read camera config to get the current good burntool state : + ## XXX This is extremely slow. Any better way to do this? + ## my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -"; + ## ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + ## run(command => $camera_config_cmd, verbose => 0); + ## unless ($success) { + ## $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + ## &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + ## } + ## + ## my $camData = $mdcParser->parse(join "", @$stdout_buf) or + ## &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + + ## XXX short term hack until we have a C-based parser + my $burntoolStateGood = 14; + my $burntoolStateGoodUpdate = 13; + ## foreach my $camEntry (@$camData) { + ## if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") { + ## $burntoolStateGood = $camEntry->{value}; + ## } + ## if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD.UPDATE") { + ## $burntoolStateGoodUpdate = $camEntry->{value}; + ## } + ## } if ($run_state eq 'new') { @@ -329,4 +332,14 @@ # get the UNIX version of the (possible) neb: or path: filename my $uriReal = $ipprc->file_resolve( $uri ); + # Catch errors here + if (!$uriReal) { + my $repair_cmd = "$nebrepair $uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + $uriReal = $ipprc->file_resolve( $uri ); + } + &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal; @@ -335,4 +348,13 @@ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); unless ($success) { + # Catch errors here + my $repair_cmd = "$nebrepair $uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose); + } + unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); @@ -347,5 +369,14 @@ $burntoolTable_uri =~ s/fits$/burn.tbl/; $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + if ((!$burntoolTable_uriReal)||(!($ipprc->file_exists($burntoolTable_uri)))) { + my $repair_cmd = "$nebrepair $burntoolTable_uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $uri $repair_error_code", $exp_id,$chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri ); + } unless ($ipprc->file_exists($burntoolTable_uri)) { + # Catch errors here &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR); } @@ -439,5 +470,5 @@ $command .= " -Db PPIMAGE:PHOTOM FALSE"; } - if ($run_state eq "new") { + if ($run_state eq "new" or $do_stats) { $command .= " -recipe PPSTATS CHIPSTATS"; $command .= " -stats $outputStats"; @@ -450,5 +481,5 @@ $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id; $command .= " -source_id $source_id" if defined $source_id; - $command .= " -tracedest $traceDest -log $logDest"; + $command .= " -tracedest $traceDest"; $command .= " -dbname $dbname" if defined $dbname; @@ -531,14 +562,23 @@ # Add the processed file to the database -unless ($no_update) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); - exit($error_code); - } -} else { - print "skipping command: $command\n"; +my $tries = 0; +while (1) { + $tries++; + unless ($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -addprocessedimfile: $error_code\n"); + exit($error_code) if $tries >= 3; + warn("Waiting 10 seconds to try again\n"); + sleep 10; + } else { + last; + } + } else { + print "skipping command: $command\n"; + last; + } } @@ -726,4 +766,16 @@ &my_die("Couldn't find expected output file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); } + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + ##### if ($replicate and $neb) { Index: branches/meh_branches/ppstack_test/ippScripts/scripts/diff_skycell.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/diff_skycell.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/diff_skycell.pl (revision 33415) @@ -40,5 +40,4 @@ my ($skycell_id, $diff_skyfile_id); my ($use_convolved); -my $zaplog = 0; GetOptions( @@ -57,5 +56,4 @@ 'no-op' => \$no_op, # Don't do any operations? 'redirect-output' => \$redirect, - 'zaplog' => \$zaplog, # delete any previous log file ) or pod2usage( 2 ); @@ -79,15 +77,9 @@ # XXX camera is not known here; cannot use filerules... -# bills 2011-03-07 This is unfortunate because it prevents us from using prepare_output ... -# my $logDest = $ipprc->filename("LOG.EXP", $outroot); my $logDest = "$outroot.log"; if ($run_state eq 'update') { $logDest .= '.update'; - # ... well we can just do the dirty work here - $ipprc->kill_file($logDest); -} elsif ($zaplog) { - $ipprc->kill_file($logDest); -} -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; +} +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect; my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); @@ -285,8 +277,16 @@ $outputStats = prepare_output("SKYCELL.STATS.UPDATE", $outroot, 1); $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot); - if (!$ipprc->file_exists($configuration)) { - print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n"; - $configuration = undef; - } + # XXX: Work around problem with updating old diff runs. Their config files are incompatible + # with the current code. + if ($camera ne 'GPC1' or $diff_id >= 88268) { + if (!$ipprc->file_exists($configuration)) { + print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n"; + $configuration = undef; + } + } else { + print STDERR "WARNING: Using new recipes because config dump file is too old\n"; + $configuration = undef; + } + # use WARPSTATS if we're updating, as we don't care about the new stuff from the STACK and DIFF STATS recipes $recipe_ppstats = 'WARPSTATS'; @@ -468,4 +468,18 @@ &my_die("Couldn't find expected output file: $file", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + + if ($replicate and $neb) { $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR); Index: branches/meh_branches/ppstack_test/ippScripts/scripts/dist_bundle.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/dist_bundle.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/dist_bundle.pl (revision 33415) @@ -194,4 +194,8 @@ # XXX: perhaps only do this for stages where we know that this happens next if $file_rule =~ /STATS/; + # don't fail on these "non-essential files" + next if $file_rule =~ /TRACE/; + next if $file_rule =~ /LOG/; + next if $file_rule =~ /BIN/; &my_die("failed to resolve $file_name", $component, $PS_EXIT_DATA_ERROR); @@ -636,4 +640,8 @@ $file{file_rule} = $key; $file{name} = $val; + if ($val eq "STDERR" or $val eq "STDOUT") { + print STDERR "Skipping $key because filename is $val\n"; + next; + } push @file_list, \%file; } Index: branches/meh_branches/ppstack_test/ippScripts/scripts/dist_component.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/dist_component.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/dist_component.pl (revision 33415) @@ -37,5 +37,5 @@ # Parse the command-line arguments my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean, $alt_path_base); -my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $exp_type); +my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $exp_type, $rerun); my ($dbname, $save_temps, $verbose, $no_update, $logfile); @@ -57,4 +57,5 @@ 'outdir=s' => \$outdir, # "directory" for outputs 'clean' => \$clean, # create clean distribution + 'rerun' => \$rerun, # recreate an existing component and update the database 'save-temps' => \$save_temps, # Save temporary files? 'dbname=s' => \$dbname, # Database name @@ -76,5 +77,5 @@ my $ipprc = PS::IPP::Config->new($camera); # IPP configuration -$ipprc->redirect_output($logfile) if $logfile; +$ipprc->redirect_to_logfile($logfile) if $logfile; my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); @@ -146,5 +147,12 @@ { - my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir"; + my $command; + if (!$rerun) { + $command = "$disttool -addprocessedcomponent"; + } else { + $command = "$disttool -updateprocessedcomponent"; + } + + $command .= " -dist_id $dist_id -component $component -outdir $outdir"; $command .= " -name $file_name -bytes $bytes -md5sum $md5sum"; $command .= " -dbname $dbname" if defined $dbname; @@ -175,5 +183,10 @@ $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; - my $command = "$disttool -addprocessedcomponent"; + my $command; + if (!$rerun) { + $command = "$disttool -addprocessedcomponent"; + } else { + $command = "$disttool -updateprocessedcomponent"; + } $command .= " -dist_id $dist_id"; $command .= " -component $component"; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/dist_defineruns.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/dist_defineruns.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/dist_defineruns.pl (revision 33415) @@ -88,10 +88,19 @@ push @stages, $stage; } else { - @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff sky); + @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle); } foreach my $stage (@stages) { foreach my $label (@labels) { - my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label"; + my $single; + my $cmdstage; + if ($stage eq "skysingle") { + $cmdstage = "sky"; + $single = 1; + } else { + $cmdstage = $stage; + } + my $command = "$disttool -definebyquery -stage $cmdstage -workdir $workdir -label $label"; + $command .= " -singlefilter" if $single; $command .= " -no_magic" if $no_magic; $command .= " -pretend" if $no_update; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/dist_fix_file.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/dist_fix_file.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/dist_fix_file.pl (revision 33415) @@ -0,0 +1,234 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +#print "\n\n"; +#print "Starting script $0 on $host at $date\n\n"; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename ); +use Digest::MD5::File qw( file_md5_hex ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); + +use DBI; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $disttool = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1); +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $dist_component = can_run('dist_component.pl') or (warn "Can't find dist_component.pl" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# Parse the command-line arguments +my ($dist_id, $camera, $component ); +my ($outdir, $run_state, $data_state, $magicked, $poor_quality, $exp_type, $rerun); +my ($dbname, $save_temps, $verbose, $no_update, $logfile); + +$camera = 'GPC1'; + +GetOptions( + 'dist_id=s' => \$dist_id, # distribution run identifier + 'component=s' => \$component, # the class_id or skycell_id + 'camera=s' => \$camera, # the class_id or skycell_id + 'save-temps' => \$save_temps, # Save temporary files? + 'dbname=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print stuff? + 'no-update' => \$no_update, # Don't update the database? + 'logfile=s' => \$logfile, + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dist_id --camera --component", + -exitval => 3) unless + defined $dist_id and + defined $camera and + defined $component; + +my $ipprc = PS::IPP::Config->new($camera); # IPP configuration +$ipprc->redirect_to_logfile($logfile) if $logfile; + +my $dbh = getDBHandle() unless $no_update; + +my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR"); + +$temproot = "/tmp" if !defined $temproot; + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +$disttool .= " -dbname $dbname" if $dbname; +$regtool .= " -dbname $dbname" if $dbname; + +my $comp = get_distcomponent($dist_id, $component); + +my $stage = $comp->{stage}; +my $stage_id = $comp->{stage_id}; +my $no_magic = $comp->{no_magic}; +my $path_base; +my $stage_args = ""; +if ($stage eq 'raw') { + # need path_base (get from uri) + # exp_type + # no_magic + # camera path base if !$no_magic + my $command = "$regtool -processedimfile -exp_id $stage_id -class_id $component"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code); + } + + my $metadata = $mdcParser->parse (join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + my $results = parse_md_list($metadata); + + if ((scalar @$results) != 1) { + my $n = scalar @$results; + &my_die("Unexected number of results from regtool -processedimfile: $n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + } + my $rawfile = $results->[0]; + $path_base = $rawfile->{uri}; + $path_base =~ s/.fits//; + my $exp_type = $rawfile->{exp_type}; + if (!$no_magic) { + die "don't know how to create magicked raw bundles yet\n"; + } + $stage_args .= " --exp_type $exp_type --chip_path_base NULL"; + $stage_args .= " --no_magic" if $no_magic; +} else { + + die "not ready for stage $stage yet\n"; +} +my $dist_cmd = "$dist_component --rerun --camera $camera --dist_id $dist_id --stage $stage --stage_id $stage_id --component $component --outdir $comp->{outdir} --path_base $path_base $stage_args"; + +$dist_cmd .= " --dbname $dbname" if $dbname; +$dist_cmd .= " --verbose" if $verbose; +$dist_cmd .= " --no-update" if $no_update; + +print "$dist_cmd\n"; + +$comp = get_distcomponent($dist_id, $component); + +my $bytes = $comp->{bytes}; +my $md5sum = $comp->{md5sum}; +my $name = $comp->{name}; + + +my $query = "UPDATE dsFile join dsFileset using(fileset_id) join dsProduct USING(prod_id) SET dsFile.bytes =?, dsFile.md5sum = ? WHERE prod_name = ? AND fileset_name = ? AND file_name = ?"; + +my $stmt = $dbh->prepare($query); + +my $filesets = get_filesets($dist_id); + +foreach my $fs (@$filesets) { + my $product = $fs->{product}; + my $fileset = $fs->{fileset}; + $stmt->execute($bytes, $md5sum, $product, $fileset, $name) or + &my_die("failed to update dsFile for $product $fileset $name\n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + print "Updated dsFile for $product $fileset $name $bytes $md5sum\n"; +} + + +exit 0; + +sub getDBHandle { + my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBSERVER"); + my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBUSER"); + my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBPASSWORD"); + my $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DS_DBNAME"); + + &my_die("database configuration not set up", $dist_id, $component, $outdir, $PS_EXIT_UNKNOWN_ERROR) unless defined($dbserver) and defined($dbuser) + and defined($dbpassword) and defined($dbname); + + my $dsn = "DBI:mysql:host=$dbserver;database=$dbname"; + + my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) + or die "Cannot connect to database $dbname on $dbserver.\n"; + + return $dbh; +} + + + +sub get_distcomponent { + my $dist_id = shift; + my $component = shift; + my $command = "$disttool -processedcomponent -dist_id $dist_id -component $component"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code); + } + + my $metadata = $mdcParser->parse (join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + my $results = parse_md_list($metadata); + + if ((scalar @$results) != 1) { + my $n = scalar @$results; + &my_die("Unexected number of results from disttool -processedcomponent: $n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + } + return $results->[0]; +} + +sub get_filesets { + my $dist_id = shift; + my $command = "$disttool -listfilesets -dist_id $dist_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code); + } + + my $output = join "", @$stdout_buf; + if (!$output) { + return undef; + } + my $metadata = $mdcParser->parse ($output) or + &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR); + + my $results = parse_md_list($metadata); + + return $results +} + + +### Pau. +sub my_die +{ + my $msg = shift; # Warning message on die + my $dist_id = shift; # distRun.dist_id + my $component = shift; # class_id, skycell_id, or exposure + my $outdir = shift; # output directory + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + exit $exit_code; +} + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_cleanup.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_cleanup.pl (revision 33415) @@ -218,4 +218,6 @@ $command .= " -dbname $dbname" if defined $dbname; +if (0) { + # XXX Don't set components to error cleaned anymore my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -224,4 +226,5 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } +} # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. Index: branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_filename.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_filename.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_filename.pl (revision 33415) @@ -10,4 +10,10 @@ #print "$ENV{'PATH'}\n"; #print "$ENV{'PERL5LIB'}\n"; + +# CZW: This is a horrible hack, but I don't want to have to debug all of ippMonitor to figure out why it's not working. +unless (defined($ENV{'NEB_SERVER'})) { + $ENV{'NEB_SERVER'} = 'http://nebserver.ipp.ifa.hawaii.edu:80/nebulous'; +} + use PS::IPP::Config; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_serial_inject_into_nebulous.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_serial_inject_into_nebulous.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/ipp_serial_inject_into_nebulous.pl (revision 33415) @@ -0,0 +1,169 @@ +#!/usr/bin/env perl + +# this program injects a list of single-file exposures into the db, +# taking the filename (without .fits) as the exp_tag. the user +# supplies a temporary telescope and camera name. these are used +# for informational purposes only until the registration step can +# determine the true telescope and camera name from the image +# headers. + +# this program should not fail because of the data format or the +# configuration, except for the very basic database setup. + +use warnings; +use strict; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use PS::IPP::Config; + +my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Parse the command-line arguments +my ($camera, $telescope, $workdir, $reduction, $dvo_db, $tess_id, $end_stage, $label, $dbname, $no_op, $help); +GetOptions('camera|i=s' => \$camera, # user-supplied camera name + 'telescope|t=s' => \$telescope, # user-supplied telescope name + 'workdir|w=s' => \$workdir, # working directory for output files + 'reduction=s' => \$reduction, # user-supplied camera name + 'dvodb=s' => \$dvo_db, # target dvo database + 'tess_id=s' => \$tess_id, # tessalation for warping + 'end_stage=s' => \$end_stage, # stop processing at this step + 'label=s' => \$label, # set chip label + 'dbname|d=s' => \$dbname, # Database name + 'no-op' => \$no_op, # pretend but don't actually inject + 'help' => \$help # give help listing +) or pod2usage( 2 ); + +pod2usage( -msg => "inject one or many files into the IPP pipeline database", + -exitval => 2) if + defined $help; + +pod2usage( -msg => "Usage: $0 --telescope (name) --camera (name) [--workdir path] [--reduction class] [--dvodb db] [--tess_id tess] [--end_stage stage] [--label label] [--dbname dbname] (files)", + -exitval => 2 ) if + scalar @ARGV == 0; + +pod2usage( -msg => "Required options: --telescope (name) --camera (name)", + -exitval => 3) unless + defined $telescope and + defined $camera; + +my $pxinject = can_run('pxinject') or die "Can't find pxinject\n"; + +# if workdir is not defined, assign the current path +# XXX we need to handle relative paths for workdir (not allowed) +if (! $workdir) { + $workdir = File::Spec->rel2abs( "." ); +} + +my $num = 0; +foreach my $file ( @ARGV ) { + # check for file existence + if (! -e $file) { die "file $file not found\n"; } + my $absfile = File::Spec->rel2abs( $file ); + + #copy into nebulous + my $neb_part=$absfile; + $neb_part =~s/.*\///g; + my $base = $neb_part; + $base =~s/.fits//; + + print "$workdir/$base/$neb_part\n"; + + + `neb-touch $workdir/$base/$neb_part`; + my @paths = `neb-locate --path $workdir/$base/$neb_part`; + foreach my $path (@paths) { + chomp $path; + my @cperr = `cp $absfile $path 2>&1`; + if ($#cperr > 0) { + print "error copying $absfile $path\n"; + } + } + `neb-replicate $workdir/$base/$neb_part --set_copies 2`; + my $numofcopies = `neb-stat $workdir/$base/$neb_part | grep available`; + $numofcopies =~ s/available instances: //; + if ($numofcopies < 2) { + print "problem: less than 2 copies of $workdir/$base/$neb_part\n"; + } + + inject("$workdir/$base/$neb_part", $workdir, $dbname, $telescope, $camera); + $num ++; +} + +print "$num files injected.\n"; + +sub inject +{ + my $absfile = shift; # absolute path for this file + my $workdir = shift; # absolute path for output directory + my $dbname = shift; # IPP database to use + my $telescope = shift; # user-specified telescope + my $camera = shift; # user-specified camera + + # XXX provide an option for an alternative extension + my ( $vol, $path, $name ) = File::Spec->splitpath( $absfile ); + my ( $exp_name ) = $name =~ /(.*)\.(fits|fit|fts)(|.gz)/; + $exp_name =~ s|\s|_|g; + + my $relfile = $ipprc->convert_filename_relative( $absfile ); + + # the telescope, instrument, and exp_name used here are temporary : register replaces them with the true values + my $command_exp = "$pxinject -newExp"; + $command_exp .= " -tmp_exp_name $exp_name"; + $command_exp .= " -tmp_inst $camera"; + $command_exp .= " -tmp_telescope $telescope"; + $command_exp .= " -workdir $workdir"; + $command_exp .= " -reduction $reduction" if defined $reduction; + $command_exp .= " -dvodb $dvo_db" if defined $dvo_db; + $command_exp .= " -tess_id $tess_id" if defined $tess_id; + $command_exp .= " -end_stage $end_stage" if defined $end_stage; + $command_exp .= " -label $label" if defined $label; + $command_exp .= " -dbname $dbname" if defined $dbname; + + my $exp_id = 0; + unless ($no_op) { + my ( $success_exp, $error_code_exp, $full_buf_exp, $stdout_buf_exp, $stderr_buf_exp ) = + run( command => $command_exp, verbose => 1 ); + die "Unable to inject $exp_name: $error_code_exp\n" if not $success_exp; + + my @line = split(/\s+/, $$stdout_buf_exp[0]); # The output line, containing the exposure tag + $exp_id = $line[2]; # The exposure tag + } else { + print "skipping command: $command_exp\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_imfile = "$pxinject -newImfile"; + $command_imfile .= " -exp_id $exp_id"; + $command_imfile .= " -tmp_class_id fpa"; + $command_imfile .= " -uri '$relfile'"; + $command_imfile .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_imfile, $error_code_imfile, $full_buf_imfile, $stdout_buf_imfile, $stderr_buf_imfile ) = run( command => $command_imfile, verbose => 1 ); + die "Unable to inject $exp_name imfile: $error_code_imfile\n" if not $success_imfile; + } else { + print "skipping command: $command_imfile\n"; + } + + # the class_id used here is temporary : register replaces it with the true class_id + my $command_update = "$pxinject -updatenewExp"; + $command_update .= " -exp_id $exp_id"; + $command_update .= " -state run"; + $command_update .= " -dbname $dbname" if defined ($dbname); + + unless ($no_op) { + my ( $success_update, $error_code_update, $full_buf_update, $stdout_buf_update, $stderr_buf_update ) = run( command => $command_update, verbose => 1 ); + die "Unable to update $exp_name: $error_code_update\n" if not $success_update; + } else { + print "skipping command: $command_update\n"; + } + + return 1; +} + + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/lap_science.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/lap_science.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/lap_science.pl (revision 33415) @@ -27,4 +27,5 @@ my ( $camera, $dbname); my ( $lap_id ); +my ( $queue_list ); my ( $chip_mode, $monitor_mode, $cleanup_mode); @@ -32,5 +33,5 @@ # Global configuration constants that probably should be read from elsewhere. my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. -my $minimum_stack_inputs = 6; # We can avoid magicking stack inputs if we have more than this number. +my $minimum_stack_inputs = 4; # We can avoid magicking stack inputs if we have more than this number. GetOptions( @@ -44,4 +45,5 @@ 'lap_id=s' => \$lap_id, + 'queue_list=s' => \$queue_list, 'chip_mode' => \$chip_mode, @@ -200,4 +202,122 @@ return($exposure); } + +sub remake_this_exposure_by_update { + my $exposure = shift; + + my @utctime = gmtime(); + $utctime[5] += 1900; + $utctime[4] += 1; + + my $label = $exposure->{label}; + + my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); + my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; + my $data_group = "${label}.${date}"; + + my $chiptool_info_cmd = "chiptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} "; + $chiptool_info_cmd .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_info_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", $exposure->{exp_id}, $data_group); + } + my $chips = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from chiptool -listrun", $exposure->{exp_id}, $data_group); + # There should be only one. + my $chip = ${ $chips }[0]; + my $chip_magicDS_id = 0; + if ($chip) { + $chip_magicDS_id = $chip->{magic_ds_id}; + } + if ($chip_magicDS_id == 0) { +# return(&remake_this_exposure($exposure)); + } + + my $warptool_info_cmd = "warptool -listrun -exp_id $exposure->{exp_id} -chip_id $exposure->{chip_id} "; + $warptool_info_cmd .= " -dbname $dbname " if defined $dbname; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_info_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -listrun: $error_code", $exposure->{exp_id}, $data_group); + } + my $warps = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from warptool -listrun", $exposure->{exp_id}, $data_group); + # There should be only one. + my $warp = ${ $warps }[0]; + my $warp_id = 0; + if ($warp) { + $warp_id = $warp->{warp_id}; + } + else { + $exposure->{data_state} = 'drop'; + return($exposure); + } + if ($warp_id == 0) { + # Handle this correctly. Probably want to drop the exposure here. + $exposure->{data_state} = 'drop'; + return($exposure); + } + + if (($chip->{state} eq 'goto_cleaned')|| + ($warp->{state} eq 'goto_cleaned')|| + ($chip->{dsRun_state} eq 'goto_cleaned')) { + $exposure->{data_state} = 'pending_update'; + &update_this_exposure($exposure); + return($exposure); + } + + + my $chiptool_update_cmd = "chiptool -setimfiletoupdate -chip_id $exposure->{chip_id} -set_label $label"; + $chiptool_update_cmd .= " -dbname $dbname " if defined $dbname; + my $magicDS_update_cmd = "magicdstool -setfiletoupdate -magic_ds_id $chip_magicDS_id -set_label $label"; + $magicDS_update_cmd .= " -dbname $dbname " if defined $dbname; + my $warptool_update_cmd = "warptool -setskyfiletoupdate -warp_id $warp_id -set_label $label"; + $warptool_update_cmd .= " -dbname $dbname " if defined $dbname; + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $chiptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -setimfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } +# if ($chip_magicDS_id != 0) { +# ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = +# run(command => $magicDS_update_cmd, verbose => $verbose); +# unless ($success) { +# $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); +# &my_die("Unable to perform magicdstool -setfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); +# } +# } + + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $warptool_update_cmd, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -setskyfiletoupdate: $error_code", $exposure->{exp_id}, $data_group); + } + + $exposure->{active} = 1; + return($exposure); +} + +sub determine_if_can_update { + my $exposure = shift; + + if (S64_IS_NOT_NULL($exposure->{chip_id})) { + if (($exposure->{chip_state} eq 'cleaned')|| + ($exposure->{chip_state} eq 'goto_cleaned')|| + ($exposure->{chip_state} eq 'error_cleaned')) { + return(1); + } + } + + return(0); +} + # This is the "user level" subroutine. @@ -240,5 +360,7 @@ my $comment = $exposure->{comment}; - # This is a hack to fix old exposures that have no object. + my $updateable = determine_if_can_update($exposure); + + # This is a hack to fix old exposures that have no usable object/comment data. unless(defined($comment)) { $comment = ''; @@ -247,22 +369,41 @@ if ($comment =~ /3pi_/) { $object = $comment; - $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/$1/; + $object =~ s/^.*?(3pi_\d\d_\d\d\d\d).*?$/F1 $1/; } elsif ($comment =~ / ps1_/) { $object = $comment; - $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/$1/; - } - else { - $exposure->{data_state} = 'new'; + $object =~ s/^.*?(ps1_\d\d_\d\d\d\d).*$/F2 $1/; + } + elsif ($comment =~ /ThreePi.*3pi_/) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? 3pi_\d\d_\d\d\d\d) .*$/F3 $1/; + } + elsif ($comment =~ /ThreePi.*ps1_/) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? ps1_\d\d_\d\d\d\d) .*$/F4 $1/; + } + elsif ($comment =~ /ThreePi /) { + $object = $comment; + $object =~ s/^ThreePi . (\S+? \d\d\d\d) .*$/F5 $1/; + } + elsif (($comment =~ /focus/i)|| + ($comment =~ /test/i)|| + ($comment =~ /hyster/i)|| + ($comment =~ /dither/i)|| + ($comment =~ /camera/i) + ){ + # This is junk that shouldn't exist. + $object = 'DROP'; + $exposure->{data_state} = 'drop'; $exposure->{pairwise} = 0; - $exposure->{private} = 1; - $exposure->{pair_id} = 9223372036854775807; - update_this_exposure($exposure); - $exposure = remake_this_exposure($exposure); - $counter++; + $exposure->{pair_id} = 9223372036854775807; + $exposure->{active} = 0; + &update_this_exposure($exposure); + $counter++; # To ensure everyone else is consistent next; } } + # Determine the current state of chipRuns for these exposures, and update/remake as needed. if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id if (($pairwise) && !($pair_id)) { @@ -270,18 +411,25 @@ &my_die("Exposure $exp_id for $lap_id is declared pairwise without a defined pair", $lap_id); } + $exposure->{data_state} = 'exists'; + if ($updateable) { # We know about this run, but the data needs to be regenerated. + $exposure->{data_state} = 'update'; + $exposure = remake_this_exposure_by_update($exposure); + } + &update_this_exposure($exposure); $matching{$object}{$comment} = $exp_id; $indexing{$exp_id} = $counter; $counter++; - next; } else { # We do not already have a chip_id. # Make a chipRun, and update the exposure. + $exposure->{data_state} = 'run'; $exposure = remake_this_exposure($exposure); - # Save our information for diff pairing. + &update_this_exposure($exposure); $matching{$object}{$comment} = $exp_id; $indexing{$exp_id} = $counter; $counter++; } + if ($verbose) { print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; @@ -296,5 +444,5 @@ print "$object $comment $matching{$object}{$comment} $indexing{$matching{$object}{$comment}} $exp_ids_to_diff[-1]\n"; } - @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; + @exp_ids_to_diff = sort { $indexing{$a} <=> $indexing{$b} } @exp_ids_to_diff; # This is effectively a sort by dateobs. if (( $#exp_ids_to_diff + 1) % 2 != 0) { # We have an odd number of exposures, even after comment filtering @@ -309,14 +457,17 @@ my $exp_B = ${ $exposures }[$indexing{$exp_id_B}]; print "$exp_A $exp_B $exp_id_A $exp_id_B $indexing{$exp_id_A} $indexing{$exp_id_B}\n"; - $exp_A->{pairwise} = 1; - $exp_A->{private} = 0; - $exp_A->{pair_id} = $exp_B->{chip_id}; - - $exp_B->{pairwise} = 1; - $exp_B->{private} = 0; - $exp_B->{pair_id} = $exp_A->{chip_id}; - - if ($verbose) { - print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n"; + if ($exp_A->{diff_id} == $exp_B->{diff_id}) { + $exp_A->{pairwise} = 1; + $exp_A->{pair_id} = $exp_B->{chip_id}; + + $exp_B->{pairwise} = 1; + $exp_B->{pair_id} = $exp_A->{chip_id}; + if ($verbose) { + print "LAP_DIFFS: $object: $exp_A->{exp_id} and $exp_B->{exp_id} are a pair\n"; + } + } + else { + $exp_A->{pairwise} = 0; + $exp_B->{pairwise} = 0; } } @@ -331,8 +482,5 @@ $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. } - if (!($exposure->{pairwise})) { - $exposure->{private} = 1; # If this isn't being pairwise diffed, it needs to be private - } - + update_this_exposure($exposure); } @@ -403,4 +551,8 @@ $complete_fstacks++; } + elsif (($stack->{final_state})&& + ($stack->{final_state} eq 'drop')) { + $complete_fstacks++; + } elsif (($stack->{final_state})&&($stack->{final_state} eq 'new')&& ($stack->{final_fault} >= 4)&&($stack->{final_fault} != 32767)) { @@ -457,6 +609,6 @@ # Things I want to know before I'm through # my $needs_qstack = 0; - my $needs_something_remade = 0; - my $needs_something_private = 0; + my $needs_something_remade = 0; # I never need something remade, because we're committed to one-exp=one-chip + my $needs_something_private = 0; # I never need something private, as that is calculated by the difference engine. # my $can_qstack = 0; my $have_diff = 0; @@ -468,5 +620,4 @@ my $are_warped = 0; my $are_magicked = 0; - foreach my $exposure (@$exposures) { $total_exposures++; @@ -480,5 +631,4 @@ else { # We claimed to be pairwise, but do not have a valid pair_id. $exposure->{pairwise} = 0; - $exposure->{private} = 1; &update_this_exposure($exposure); } @@ -487,22 +637,25 @@ if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. $are_warped++; -# $can_qstack ++; $can_diff ++; $have_diff ++; $are_magicked ++; -# $can_fstack ++; + next; + } + + if ($exposure->{data_state} eq 'pending_update') { + $exposure->{data_state} = 'update'; + $exposure = &remake_this_exposure_by_update($exposure); + &update_this_exposure($exposure); next; } if ($exposure->{private}) { # I've declared this exposure private to this lapRun. -# $needs_qstack++; push @lonely_exposures, $exposure; -# $lonely_exposures++; } - if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun - $needs_something_remade = 1; - $exposure = remake_this_exposure($exposure); - } +# if ($exposure->{needs_remade}) { # This does the check that private = false for other lapRun +# $needs_something_remade = 1; +# $exposure = remake_this_exposure($exposure); +# } # Do quality checks here my $is_bad_quality = 0; @@ -536,18 +689,14 @@ ($exposure->{diffRun_state} eq 'full')) { $are_warped++; -# $needs_qstack++; - $needs_something_private = 1; if ($companion) { - $companion->{private} = 1; $companion->{pairwise} = 0; &update_this_exposure($companion); - push @lonely_exposures, $companion; -# $lonely_exposures++; +# push @lonely_exposures, $companion; } - $exposure->{private} = 1; $exposure->{pairwise} = 0; } $exposure->{data_state} = 'drop'; &update_this_exposure($exposure); + next; } @@ -558,12 +707,28 @@ # $can_qstack ++; $can_diff ++; - } - if (($exposure->{magicked}&& - &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. + $exposure->{data_state} = 'to_diff'; + } + if (($exposure->{diff_id})&& + (&S64_IS_NOT_NULL($exposure->{diff_id}))&& + (($exposure->{diffRun_state} eq 'full')|| + ($exposure->{diffRun_state} eq 'error_cleaned')|| + ($exposure->{diffRun_state} eq 'cleaned')) + ) { $are_magicked++; -# $can_fstack ++; - } - if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { $have_diff ++; +# $exposure->{data_state} = 'to_magic'; + $exposure->{data_state} = 'full'; + } + # We no longer need to care about magic +# if (($exposure->{magicked})&& +# ($exposure->{warpRun_state})&& +# ($exposure->{warpRun_state} eq 'full')&& +# (&S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. +# $are_magicked++; +# $exposure->{data_state} = 'full'; +# # $can_fstack ++; +# } + unless ($debug) { + &update_this_exposure($exposure); } } @@ -575,13 +740,10 @@ print "STATUS: HAVE_FSTACK: $have_fstack\n"; -# print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; print "STATUS: ARE WARPED: $are_warped\n"; print "STATUS: ARE MAGICKED: $are_magicked\n"; -# print "STATUS: CAN_QSTACK: $can_qstack\n"; print "STATUS: CAN_DIFF: $can_diff\n"; print "STATUS: HAVE_DIFF: $have_diff\n"; -# print "STATUS: CAN_FSTACK: $can_fstack\n"; print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; @@ -595,5 +757,8 @@ if ($defined_fstack == 0) { # We have not yet made any stacks. print "STATUS: We can attempt to queue deep final stacks.\n"; - my $Nstacks_made = &queue_muggle_finalstack($exposures); + my $Nstacks_made; + unless ($debug) { + $Nstacks_made = &queue_muggle_finalstack($exposures); + } print "STATUS: Made $Nstacks_made final stacks.\n"; exit($Nstacks_made); @@ -603,12 +768,15 @@ if ($have_fstack == $defined_fstack) { # We have made all the stacks we have asked for (so far) if ($are_magicked < $total_exposures) { # But we have not yet made diffs +# if ($have_diff < $total_exposures) { # We have not yet made diffs if (($can_diff == $total_exposures)&& # And we have exposures that should be diffed. ($have_diff < $can_diff)) { # and we haven't made them all yet, either. print "STATUS: Dropping unpaired exposures and making pairwise diffs.\n"; - foreach my $exposure (@lonely_exposures) { - $exposure->{data_state} = 'drop'; - &update_this_exposure($exposure); +# foreach my $exposure (@lonely_exposures) { +# $exposure->{data_state} = 'drop'; +# &update_this_exposure($exposure); +# } + unless ($debug) { + &queue_diffs($exposures); } - &queue_diffs($exposures); exit(0); } @@ -621,24 +789,30 @@ } elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic - print "STATUS: We can create all remaining stacks possible.\n"; - my $Nstacks_made = &queue_magic_finalstack($exposures); - if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. + print "STATUS: We would create all remaining stacks possible, but no longer require magic.\n"; +# my $Nstacks_made = -1; +# unless ($debug) { +# $Nstacks_made = &queue_muggle_finalstack($exposures); +# } +# if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. print "STATUS: No more stacks generated. Deactivating exposures for this complete lapRun.\n"; - &deactivate_exposures($exposures); - $command = "$laptool -updaterun -lap_id $lap_id"; - $command .= " -dbname $dbname " if defined $dbname; - $command .= " -set_state full "; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); - } + unless ($debug) { + &deactivate_exposures($exposures); + + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state full "; + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } +# } exit(0); } - else { - print "STATUS: $Nstacks_made stacks made.\n"; - exit(0); - } +# else { +# print "STATUS: $Nstacks_made stacks made.\n"; +# exit(0); +# } } else { # What? @@ -842,5 +1016,5 @@ foreach $exposure (@$exposures) { if (($exposure->{data_state} ne 'drop')&& - (S64_IS_NOT_NULL($exposure->{magicked}))&& +# (S64_IS_NOT_NULL($exposure->{magicked}))&& (S64_IS_NOT_NULL($exposure->{warp_id}))) { $warps .= " -warp_id $exposure->{warp_id} "; @@ -924,5 +1098,5 @@ next; } - if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # Not sure how this would happen, but still. ## This happens when we inherit a complete exposure. + if ($exposure->{diff_id}&&S64_IS_NOT_NULL($exposure->{diff_id})) { # This happens when we inherit a complete exposure. next; } @@ -964,9 +1138,21 @@ } else { # warp-qstack - next; # We no longer wish to make these, and should never get here. - $command .= '-pretend'; -# $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; -# $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; -# $already_queued{$exposure->{warp_id}} = 1; + # We need to decide if we can make a warp-stack diff: + if (&can_warp_stack_diff_be_made($exposure)) { +# next; +# $command .= '-pretend'; + $command .= " -definewarpstack -available -good_frac 0.2 "; + $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; + $already_queued{$exposure->{warp_id}} = 1; + $exposure->{private} = 0; + &update_this_exposure($exposure); + } + else { + $exposure->{private} = 1; + $exposure->{data_state} = 'drop'; + $already_queued{$exposure->{warp_id}} = 1; + &update_this_exposure($exposure); + next; + } } @@ -1006,10 +1192,52 @@ } +sub can_warp_stack_diff_be_made { + my $exposure = shift; + my $lap_id = $exposure->{lap_id}; + my $warp_id = $exposure->{warp_id}; + + my $command = "$laptool -diffcheck -lap_id $lap_id -warp_id $warp_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -diffcheck: $error_code", $lap_id, $warp_id); + } + if (@$stdout_buf == 0) { + return(0); + } + my $skycells = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from laptool -diffcheck", $lap_id, $warp_id); + foreach my $skycell (@$skycells) { + unless (defined($skycell->{stack_state})) { + return(0); + } + if ($skycell->{stack_state} ne 'full') { + return(0); + } + if ($skycell->{warp_state} ne 'full') { + return(0); + } + unless (S64_IS_NOT_NULL($skycell->{stack_id})) { + return(0); + } + } + return(1); +} + + # Deactivate all exposures in this run. sub deactivate_exposures { my $exposures = shift; foreach my $exposure (@$exposures) { - $exposure->{active} = 0; - update_this_exposure($exposure); + if ($exposure->{private} == 1) { # This can probably be relaxed since we can update an undestreaked warp. + $exposure->{active} = 1; + } + else { + $exposure->{active} = 0; + } + update_this_exposure($exposure); } } @@ -1022,4 +1250,54 @@ sub cleanup_mode { my $lap_id = shift; + + if (defined($queue_list)) { + my $successful = 1; + open(Q,$queue_list) or ($successful = 0); + if ($successful == 1) { + while() { + chomp; + my $cmd = $_; + my ($projection_cell,$seq_id,$filter,$label); + $projection_cell = $cmd; + $projection_cell =~ s/.*?-projection_cell (skycell.\w+?) .*/$1/; + $seq_id = $cmd; + $seq_id =~ s/.*-seq_id (\d+?) .*/$1/; + $filter = $cmd; + $filter =~ s/.*-filter (\w\.00000) .*/$1/; + if ($filter =~ /\s+?/) { + $filter =~ s/.*-filter (\w\.00000)$/$1/; + } + $label = $cmd; + $label =~ s/.*-label (.+?) .*/$1/; + + my $response; + chomp($response = + `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`); + my $state = (split /\s+/, $response)[5]; + + unless (defined($state)) { + if ($verbose) { + print "Queuing: $cmd\n"; + } + system($cmd); + + my $i = 0; + do { + sleep(5); + chomp($response = + `laptool -dbname gpc1 -listrun -projection_cell $projection_cell -seq_id $seq_id -filter $filter -label $label -simple`); + $state = (split /\s+/, $response)[5]; + $i++; + unless(defined($state)) { + $successful = 0; + } + } while (($state ne 'run')&&($i < 20)&&($successful)); + last; + } + } + close(Q); + } + } + my $command = "$laptool -inactiveexp -lap_id $lap_id "; $command .= " -dbname $dbname " if defined $dbname; @@ -1033,4 +1311,14 @@ if (@$stdout_buf == 0) { # Nothing to do. + $command = "$laptool -updaterun -lap_id $lap_id"; + $command .= " -dbname $dbname " if defined $dbname; + $command .= " -set_state done "; + ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); + } + return(0); } @@ -1043,7 +1331,9 @@ 'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@', 'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@', + 'magictool -dbname @DBNAME@ -updaterun -set_state full -set_label @LABEL@.old -magic_id @MAGIC_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage chip -stage_id @CHIP_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage warp -stage_id @WARP_ID@', 'magicdstool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -stage diff -stage_id @DIFF_ID@'); + foreach my $exposure (@$exposures) { if ($exposure->{is_in_use}) { @@ -1077,4 +1367,13 @@ } } + if ($command =~ /\@MAGIC_ID\@/) { + if ((S64_IS_NOT_NULL($exposure->{magicked}))&& + ($exposure->{magicked} > 0)) { + $command =~ s/\@MAGIC_ID\@/$exposure->{magicked}/; + } + else { + next; + } + } $command =~ s/\@LABEL\@/$exposure->{label}/; @@ -1139,5 +1438,5 @@ &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); } - + return(0); } @@ -1210,10 +1509,12 @@ $command .= " -set_data_state $exposure->{data_state} "; } - - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + unless ($do_nothing) { + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform laptool -updateexp: $error_code", $exposure->{lap_id}, $exposure->{proj_cell}); + } } } Index: branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak.pl (revision 33415) @@ -35,4 +35,5 @@ my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $dvoImageOverlaps = can_run('dvoImageOverlaps') or (warn "Can't find dvoImageOverlaps" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); @@ -45,4 +46,5 @@ my ($outroot, $recoveryroot, $magicked); my ($replace, $release); +my ($diff_tess_id, $mismatched_tess); my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile); @@ -60,4 +62,6 @@ 'stage_id=s' => \$stage_id, # exp_id, chip_id, warp_id, or diff_id 'component=s' => \$component, # the class_id or skycell_id + 'mismatched_tess'=> \$mismatched_tess, # true if tess_id of input does not match tess_id of the diff used for magic + 'diff_tess_id=s' => \$diff_tess_id, # tess_id of diffRun used to compute the streaks 'uri=s' => \$uri, # uri of the input image 'path_base=s' => \$path_base, # path_base of the input @@ -95,5 +99,10 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; + +if ($logfile) { + $ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Continuing script $0 on $host at $date\n\n"; +} $cam_reduction = 'DEFAULT' if !$cam_reduction or ($cam_reduction eq 'NULL'); @@ -106,8 +115,8 @@ if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") { $class_id = $component; - $skycell_args = " -class_id $component"; + $skycell_args = " -class_id"; } elsif ($stage eq "warp" or $stage eq "warp_bg") { $skycell_id = $component; - $skycell_args = " -skycell_id $component"; + $skycell_args = " -skycell_id"; } elsif ($stage eq "diff") { $skycell_id = $component; @@ -176,5 +185,7 @@ } -my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats"; +# XXX: create a DESTREAK file rule. For now just steal one that will work +my $error; +my $statsFile = $ipprc->prepare_output("PSASTRO.STATS", "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component", undef, 1, \$error) or &my_die("failed to prepare output for stats", $magic_ds_id, $component, $error); my $backup_path_base; @@ -215,41 +226,65 @@ my ($sfh, $skycell_list); if ($skycell_args) { - my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args"; - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code); - } - - my $getskycells_output = join "", @$stdout_buf; - if ($getskycells_output) { - my $metadata = $mdcParser->parse($getskycells_output) or - &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); - - my $skycells = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); - - ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps); - - foreach my $skycell (@$skycells) { - my $skycell_uri; - if ($skycell->{data_state} eq "full" and $skycell->{quality} = 0) { - $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); - } else { - # diff run must have been cleaned up, need to create this skycell file on the fly - if (!defined $temp_dir ) { - $temp_dir = tempdir( CLEANUP => !$save_temps); + my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells"; + $command .= " $skycell_args"; + $command .= " -dbname $dbname" if $dbname; + my @diff_components; + if ($mismatched_tess) { + # tessellation for this skycell does not match the tessellation used for the magic analysis + # used to calculate the streaks file + @diff_components = get_overlaps($path_base, $component, $diff_tess_id); + } else { + @diff_components = ($component); + } + + # hash of uris by skycell. + my %diff_skycells; + foreach my $component (@diff_components) { + my $this_command = "$command $component"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $this_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code); + } + + my $getskycells_output = join "", @$stdout_buf; + if ($getskycells_output) { + my $metadata = $mdcParser->parse($getskycells_output) or + &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); + + my $skycells = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); + + foreach my $skycell (@$skycells) { + my $skycell_uri; + $skycell->{quality} = undef; + # XXXXXXXX remake all skycells to avoid dead instances (ipp064) + if (0 and $skycell->{data_state} eq "full" and $skycell->{quality} == 0) { + $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base}); + my $skycell_id = $skycell->{skycell_id}; + $diff_skycells{$skycell_id} = $skycell_uri; + } else { + # diff run must have been cleaned up, need to create this skycell file on the fly + if (!defined $temp_dir ) { + $temp_dir = tempdir( CLEANUP => !$save_temps); + } + my $skycell_id = $skycell->{skycell_id}; + if (!$diff_skycells{$skycell_id}) { + $skycell_uri = "$temp_dir/$skycell_id"; + $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or + &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); + } + $diff_skycells{$skycell_id} = $skycell_uri; } - my $skycell_id = $skycell->{skycell_id}; - $skycell_uri = "$temp_dir/$skycell_id"; - $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or - &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR); } - print $sfh "$skycell_uri\n"; - } - close $sfh - } + } + } + # write the skycell list file + ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps); + foreach my $skycell_id (keys %diff_skycells) { + print $sfh $diff_skycells{$skycell_id} . "\n"; + } + close $sfh } @@ -360,10 +395,5 @@ $command .= " -chip_mask $ch_mask" if defined $ch_mask; $command .= " -weight $weight" if defined $weight; - if ((defined($sources))&&($ipprc->file_exists($sources))) { - $command .= " -sources $sources" if defined $sources; - } - else { - print "Did not add sources because they do not appear to exist. This may be an error.\n"; - } + $command .= " -sources $sources" if defined $sources; $command .= " -skycelllist $skycell_list" if defined $skycell_list; $command .= " -replace" if $replace; @@ -475,5 +505,5 @@ my $statsFlags; -if ($stage ne "camera") { +if (!$no_op and $stage ne "camera") { file_check($statsFile); { @@ -595,4 +625,35 @@ } + # @diff_components = get_overlaps($path_base, $component, $tess_id, $diff_tess_id); +sub get_overlaps { + my $path_base = shift; + my $component = shift; + my $diff_tess_id = shift; + + my $tess_dir = $ipprc->tessellation_catdir( $diff_tess_id ); + my $catdir = $ipprc->convert_filename_absolute($tess_dir); + + my $cmf = $ipprc->file_resolve("$path_base.cmf"); + &my_die("Unable to resolve $path_base.$cmf", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR) unless $cmf; + + my $command = "$dvoImageOverlaps -D CATDIR $catdir -accept-astrom $cmf"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoImageOverlaps $error_code", $magic_ds_id, $component, + $error_code); + } + my @list; + my $output = join "\n", @$stdout_buf; + if ($output) { + foreach my $line (split "\n", $output) { + my (undef, undef, $skycell_id) = split " ", $line; + push @list, $skycell_id; + } + } + return @list +} + sub my_die Index: branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak_cleanup.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak_cleanup.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak_cleanup.pl (revision 33415) @@ -61,5 +61,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $PS_EXIT_SYS_ERROR ) if $logfile; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak_revert.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak_revert.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/magic_destreak_revert.pl (revision 33415) @@ -9,6 +9,6 @@ my $host = hostname(); my $date = `date`; -print "\n\n"; -print "Starting script $0 on $host at $date\n\n"; +print STDERR "\n\n"; +print STDERR "Starting script $0 on $host at $date\n\n"; use vars qw( $VERSION ); @@ -86,5 +86,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile; &my_die("bytes and md5sum are is required for raw stage stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if ($replace and $stage eq 'raw' and (!$bytes or !$md5sum)); @@ -413,13 +413,26 @@ if ($backup_is_not_destreaked) { if ($original_is_not_destreaked) { - # this is unexpected result. throw an error so this can be checked manually - print STDERR "both files appear to not be destreaked\n"; + print STDERR "\nBoth files appear to not be destreaked.\n"; print STDERR "original: $original\n"; print STDERR "backup: $backup\n"; my $o_basename = basename($o_path); my $b_basename = basename($b_path); - if (!($o_basename =~ /SR_/) and ($b_basename =~ /SR_/)) { - print " basenames are as expected it is safe to revert\n"; + # if original file does not have SR_ in basename and the backup does then the + # files are in their original state. Just delete back up file + # otherwise go to failed_revert for manual inspection. + # XXX: We probably can get by with just checking the basename of the original file + if (!($o_basename =~ /SR_/) ) { + # if (!($o_basename =~ /SR_/) and ($b_basename =~ /SR_/)) { + print STDERR " basenames are as expected it is safe to delete backup file\n"; + if (! $ipprc->file_delete($backup, $force)) { + print STDERR "failed to delete $backup\n"; + return 0; + } + return 1; } else { + # this is an unexpected result. throw an error so this can be checked manually + print STDERR " basenames for files are as NOT as expected unsafe to revert\n"; + print STDERR " o_basename: $o_basename\n"; + print STDERR " b_basename: $b_basename\n"; return 0; } @@ -470,9 +483,7 @@ foreach my $file (@_) { next if !$file; - if ($ipprc->file_exists($file)) { - if (!$ipprc->file_delete($file)) { - &my_die("failed to delete recovery file $file", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR); - } - } + # don't care if this fails or not (it will fail if storage + # object doesn't exist + $ipprc->kill_file($file); } } @@ -500,5 +511,9 @@ my $command = "$magicdstool -updaterun"; $command .= " -magic_ds_id $magic_ds_id"; - $command .= " -set_state failed_revert"; + if ($run_state ne 'update') { + $command .= " -set_state failed_revert"; + } else { + $command .= " -set_state failed_revert_ud"; + } $command .= " -dbname $dbname" if defined $dbname; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_copy.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_copy.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_copy.pl (revision 33415) @@ -0,0 +1,189 @@ +#!/usr/bin/env perl + +#This is essentially the same as minidvodb_copy.pl - the tables are essentially identical, and the task (to rsync a copy) is essentially identical. A new script was created for rsyncing the merges in case they should diverge in the future. + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; + +my $dtime; +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $rsync = can_run('rsync') or (warn "Can't find rsync" and $missing_tools = 1); +my $mergedvodbtool = can_run('mergedvodbtool') or (warn "Can't find mergedvodbtool" and $missing_tools = 1); +my $ssh = can_run('ssh') or (warn "Can't find ssh" and $missing_tools = 1); +my $df = can_run('df') or (warn "Can't find df" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + + + +my ( $mergedvodbcopy_id, $merge_id, $mergedvodbrun_path, $mergedvodb_rsync_path, $destination_host, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodbcopy_id|w=s' => \$mergedvodbcopy_id, #minidvodb database + 'merge_id|w=s' => \$mergedvodb_id, #minidvodb_id + 'mergedvodbrun_path|w=s' => \$mergedvodbrun_path, #minidvodb_id + 'mergedvodb_rsync_path|w=s' => \$mergedvodb_rsync_path, + 'destination_host|w=s' => \$destination_host, + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodbcopy_id --mergedvodb_rsync_path --mergedvodbrun_path --destination_host", + -exitval => 3, + ) unless + defined $mergedvodbcopy_id and + defined $mergedvodb_rsync_path and + defined $mergedvodbrun_path and + defined $destination_host; + + + +my $ipprc = PS::IPP::Config->new(); + +my @df; + + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $mergedvodbcopy_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +{ #can we rsync? + + my $sizes = `du -sk $mergedvodbrun_path`; + my $total = 0; + for(split /[\r\n]+/,$sizes) # split on one or more newline characters + { + my($number,$file) = split /\t/,$_,2; # split on tab ($file not used here) + $total += $number; + } +#print 'Total: '.$total; +{ + print "Checking available diskspace on $destination_host\n\n"; + + my $command = "$ssh $destination_host df $mergedvodb_rsync_path"; + print "$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform '$ssh $destination_host df $mergedvodb_rsync_path' $error_code", $minidvodbcopy_id, $error_code); + } + + my @fullbuf= split(/\n/, @{$stdout_buf}[0]); + # print $#{$stdout_buf}; +# print "xx $#df xx"; +# print $fullbuf[0]; + if ($#fullbuf ==1 ) { +# print $fullbuf[1]; + @df = split /\s+/, $fullbuf[1]; + if ($#df == 5) { +# + #print $df[3], $df[4]; + + $df[4]=~ s/\%//; + + if ($df[4] < 97 && $df[3]-$total) { + print "\nThere is enough available space ($total < $df[3]) for the rsync\n"; + } else { + &my_die("Not enough available disk space: $df[4]% available, $df[3] on destination, $total free required for copy", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +# + + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nCan't figure out %free or available", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } + } else { + &my_die("Cannot parse df:\n@{$full_buf}[0]\nToo many lines ($#fullbuf) vs 1", $mergedvodbcopy_id, $PS_EXIT_PROG_ERROR); + } +} +} + +{#do the rsync + my $command = "$rsync -rvuaq"; + $command .= " $mergedvodbrun_path"; + $command .= " $destination_host"; + $command .= ":$mergedvodb_rsync_path"; + print "\nPerforming rsync:\n"; + print "$command\n\n"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform rsync: $error_code", $mergedvodbcopy_id, $error_code); + } + + print "Rsync complete..\n"; + +} + +{#update the database + $dtime = 86400.0*(DateTime->now->mjd - $mjd_start); + + my $command = "$mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_state full -set_dtime $dtime"; + $command .= " -dbname $dbname" if defined $dbname; + print "$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = 20; + &my_die("Unable to update database - rsync completed without error' $error_code", $mergedvodbcopy_id, $error_code); + } + +} +exit 0; + + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $mergedbvodbcopy_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodbcopy_id\n"; + +if (defined $mergedvodb_id ) { + my $command = "mergedvodbtool -updatemergedvodbcopy -mergedvodbcopy_id $mergedvodbcopy_id"; + $command .= " -set_fault $exit_code"; + $command .= " -set_dtime $dtime" if defined $dtime; + $command .= " -dbname $dbname" if defined $dbname; + print $command; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_merge.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_merge.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_merge.pl (revision 33415) @@ -0,0 +1,237 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_merge; +my $dtime_verify; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $mergedvodb_path, $merge_id, $mergedvodb, $minidvodb_path, $dbname,$camera,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'mergedvodb_path|w=s' => \$mergedvodb_path, # output DVO database + 'minidvodb_path|w=s' => \$minidvodb_path, #minidvodb database + 'mergedvodb|w=s' => \$mergedvodb, #mergedvodb + 'merge_id|w=s' => \$merge_id, #minidvodb_id + 'dbname|d=s' => \$dbname, # Database name + 'camera|w=s' => \$camera, #camera name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --merge_id -- minidvodb_path --mergedvodb_path --mergedvodb --camera", + -exitval => 3, + ) unless + defined $mergedvodb_path and + defined $minidvodb_path and + defined $mergedvodb and + defined $merge_id and + defined $camera; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $merge_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $merge_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +# convert supplied DVO database name to UNIX filename + +unless ($no_op) { + + #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge + + #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR + { + my $nothing_faulted = 0; + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -faulted -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + print "looking for faulted:\n\n$command\n\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of faulted mergedvodbs", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted + $nothing_faulted =1; + print "previous merges are okay.\n"; + } + &my_die( "Previous merges faulted, do not proceed until they are investigated:", $merge_id, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; + } + { + my $command = "$dvoverify -s $minidvodb_path"; + print "$command\n"; + my $mjd_dvoverify_start = DateTime->now->mjd; # MJD of starting script + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dvoverify: $error_code", $merge_id, $error_code); + } + $dtime_verify = 86400.0*(DateTime->now->mjd - $mjd_dvoverify_start); # MJD of starting script + print "dvoverify time $dtime_verify\n"; + } + #Merge + + my $this_is_the_first; + { + + my $mdcParser = PS::IPP::Metadata::Config->new; + my $command = "$mergetool -listmerged -mergedvodb " . $mergedvodb; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of mergedvodbruns", $merge_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if it is the first + $this_is_the_first =1; + print "listing nothing\n"; + } else { + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config", $merge_id, $PS_EXIT_PROG_ERROR); + #this fails if there is nothing listed. I checked. + my $components = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $merge_id, $PS_EXIT_PROG_ERROR); + my $comp = $$components[0]; + my $mergedvodb_path_tmp = $comp->{mergedvodb_path}; + + if (!defined($mergedvodb_path_tmp)) { + &my_die("Unable to parse mergedvodb_path", $merge_id, $PS_EXIT_PROG_ERROR); + } #but just to make sure, have it grab a minidvodb_name, to make sure it's not junk. + print "found at least 1 mergedvodbrun in merged state\n"; + $this_is_the_first = 0; + } + } + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + if (-e "$mergedvodb_path/Image.dat") { + if ($this_is_the_first == 1) { + &my_die("refusing to merge, this is the first, but files already exist in dir", $merge_id, 4); + } + $this_is_the_first =0; + } + + print "$this_is_the_first $mergedvodb_path/Image.dat\n"; + { + my $merge_command; + my $mjd_merge_start = DateTime->now->mjd; # MJD of starting script + if ($this_is_the_first) { + $merge_command = "rsync -rvat $minidvodb_path/* $mergedvodb_path"; + } else { + $merge_command = "$dvomerge $minidvodb_path into $mergedvodb_path"; + } + print "\n$merge_command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $merge_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to merge: $error_code", $merge_id, $error_code); + } + $dtime_merge = 86400.0*(DateTime->now->mjd - $mjd_merge_start); # MJD of starting script + print "merge time $dtime_merge\n"; +# } + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; +# $command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to mergedvodbprocessed: $error_code", $merge_id, $error_code); + } + } + +#} else { +# &my_die("dvodb: $mergedvodb_path not found", $merge_id, $PS_EXIT_UNKNOWN_ERROR); +} +else { + print "skipping processing for $mergedvodb_path\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $merge_id = shift; + my $exit_code = shift; # Exit code to add + + print STDERR "$msg $mergedvodb_path\n"; + +if (defined $merge_id ) { + + my $command = "mergetool -merge_id $merge_id"; + $command .= " -addmerged"; + $command .= " -mergedvodb $mergedvodb" if defined $mergedvodb; + $command .= " -fault $exit_code"; + #$command .= " -mergedvodb_path $mergedvodb_path" if defined $mergedvodb_path; + $command .= " -dtime_merge $dtime_merge" if defined $dtime_merge; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dtime_verify $dtime_verify" if defined $dtime_verify; + $command .= " -dbname $dbname" if defined $dbname; + + + + print $command; + system ($command); + } + + + + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + exit $exit_code; +} + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_queue.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_queue.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/mergedvodb_queue.pl (revision 33415) @@ -0,0 +1,164 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; +use DateTime; + +# Ideally, we want to queue new runs with mergetool -definebyquery. We can't for the survey task, because we use the value in 'site.config' to find the mergedvodb_path. I do not know how to tell pantasks how to do this, but I know how to do it in perl. + + +use DateTime::Format::Strptime; +use DateTime::Duration; +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + + +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $mergetool = can_run('mergetool') or (warn "Can't find mergetool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $outroot, $dbname, $minidvodb_group, $mergedvodb , $camera, $verbose, $no_update, + $no_op, $redirect, $save_temps); + +# we just need any old camera to be able to read site.config. I'm using SIMPLE, unless it is overridden on the commandline (it makes no difference for this case) + +$camera = 'SIMPLE'; + + +GetOptions( + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'minidvodb_group|w=s' => \$minidvodb_group, # minidvodb_group. + 'outroot|w=s' => \$outroot, # output file base name + 'mergedvodb|w=s' => \$mergedvodb, # output miniDVODB + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --mergedvodb --minidvodb_group --outroot", + -exitval => 3, + ) unless + defined $minidvodb_group and + # defined $camera and + defined $outroot and + defined $mergedvodb; + +print "camera is $camera\n"; + + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $mergedvodb , $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $minidvodb_group, $PS_EXIT_CONFIG_ERROR); +print "log file is $logDest\n"; +if ($redirect) { + $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $minidvodb_group, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + + +# Recipes to use based on reduction class + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# Output products +$ipprc->outroot_prepare($outroot); + +# the camera configurations should define the psastro output to be a single file (MEF), regardless of the inputs + my $create_new = 0; +# convert supplied DVO database name to UNIX filename +my $mergedvodbReal; +if (defined $mergedvodb) { + $mergedvodbReal = $ipprc->dvo_catdir( $mergedvodb ); # catdir for DVO + $mergedvodbReal = $ipprc->convert_filename_absolute( $mergedvodbReal ); +} else { + warn("mergedvodb undefined:\n"); + exit(4); +} +if (!defined $mergedvodbReal) { + warn("can't parse out mergedvodb_path\n"); + exit(4); +} + + + +print "mergedvodb_path = $mergedvodbReal\n"; + +unless ($no_op) { + + #create the mergedvodb entry (well, the command for it) + my $fpaCommand = "$mergetool -definebyquery"; + $fpaCommand .= " -mergedvodb $mergedvodb"; + $fpaCommand .= " -minidvodb_group $minidvodb_group"; + $fpaCommand .= " -set_mergedvodb_path $mergedvodbReal" ; + $fpaCommand .= " -dbname $dbname" if defined $dbname; + + print "running the following: \n\n"; + print "$fpaCommand\n\n"; + + unless ($no_update) { + + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $fpaCommand, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to add result to database: $error_code\n"); + exit($error_code); + } + } else { + print "skipping command: $fpaCommand\n"; + } +} + +sub my_die +{#complain if it doesn't work + my $msg = shift; # Warning message on die + my $mergedvodb = shift; # Camtool identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + carp($msg); + + exit $exit_code; +} + +END { + my $status = $?; + system("sync") == 0 + or die "failed to execute sync: $!" ; + $? = $status; +} + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_createdb.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_createdb.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_createdb.pl (revision 33415) @@ -226,5 +226,4 @@ $fpaCommand .= " -set_minidvodb_group $minidvodb_group"; $fpaCommand .= " -set_minidvodb_path $minidvodbReal" if defined $minidvodbReal; - $fpaCommand .= " -set_mergedvodb_path $dvodbReal"; $fpaCommand .= " -dbname $dbname" if defined $dbname; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_merge.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_merge.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_merge.pl (revision 33415) @@ -93,6 +93,26 @@ if (defined $mergedvodbReal) { - #this is chopped into several parts: addstar, relphot, dvoverify, merge - + #this is chopped into several parts: firstcheck, addstar, relphot, dvoverify, merge + + #first check that there were no faulted merges before. If there are, fault this merge with PS_EXIT_UNKNOWN_ERROR + { + my $nothing_faulted = 0; + my $mdcParser = PS::IPP::Metadata::Config->new; + + my $command = "$addtool -listminidvodbprocessed -faulted -minidvodb_group " . $minidvodb_group; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + &my_die( "Unable to get list of faulted minidvodbs", $minidvodb_id, $PS_EXIT_SYS_ERROR) unless $success; + if (scalar @$stdout_buf == 0 ) { #it lists nothing if nothign has faulted + $nothing_faulted =1; + print "previous merges are okay.\n"; + } + &my_die( "Previous merges faulted, do not proceed until they are investigated:", $minidvodb_group, $PS_EXIT_UNKNOWN_ERROR) unless $nothing_faulted; + + + + } #addstar Index: branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_premerge.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_premerge.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/minidvodb_premerge.pl (revision 33415) @@ -0,0 +1,174 @@ +#!/usr/bin/env perl + +use warnings; +use strict; +use Carp; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +print "\n\n"; +print "Starting script $0 on $host\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script +my $dtime_resort; +my $dtime_relphot; +my $dtime_script; + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $dvomerge = can_run('dvomerge') or (warn "Can't find dvomerge" and $missing_tools = 1); +my $addtool = can_run('addtool') or (warn "Can't find addtool" and $missing_tools = 1); +my $addstar = can_run('addstar') or (warn "Can't find addstar" and $missing_tools = 1); +my $relphot = can_run('relphot') or (warn "Can't find relphot" and $missing_tools = 1); +my $relastro = can_run('relastro') or (warn "Can't find relastro" and $missing_tools = 1); +my $dvoverify = can_run('dvoverify') or (warn "Can't find dvoverify" and $missing_tools = 1); + + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ( $minidvodb, $minidvodb_id, $minidvodb_group, $camera, $dbname,$verbose, $logfile, $no_op, $redirect, $save_temps); +GetOptions( + 'minidvodb|w=s' => \$minidvodb, #minidvodb database + 'minidvodb_id|w=s' => \$minidvodb_id, #minidvodb_id + 'minidvodb_group|w=s' => \$minidvodb_group, #minidvodb_id + 'camera|c=s' => \$camera, # Camera + 'dbname|d=s' => \$dbname, # Database name + 'verbose' => \$verbose, # Print to stdout + 'no-op' => \$no_op, # Don't do any operations? + 'logfile=s' => \$logfile, + 'save-temps' => \$save_temps, # Save temporary files? + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --minidvodb --minidvodb_id -- minidvodb_group --camera", + -exitval => 3, + ) unless + defined $minidvodb and + defined $minidvodb_id and + defined $minidvodb_group and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $minidvodb_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +if ($logfile) { + $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $minidvodb_id, $PS_EXIT_SYS_ERROR ); + print "\n\n"; + print "Starting script $0 on $host\n\n"; + print "COMMAND IS: @ARGV\n\n"; +} + +my $dtime_addstar = 0; + +unless ($no_op) { + + #this is chopped into several parts: addstar, relphot + #addstar + { + my $command = "$addstar -resort"; + $command .= " -D CAMERA $camera"; + $command .= " -D CATDIR $minidvodb"; + my $mjd_addstar_start = DateTime->now->mjd; # MJD of starting script + print "\n$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform addstar: $error_code", $minidvodb_id, $error_code); + } + $dtime_addstar = 86400.0*(DateTime->now->mjd - $mjd_addstar_start); $dtime_resort = $dtime_addstar; + # MJD of starting script + $dtime_resort = $dtime_addstar; + print "addstar -resort time $dtime_addstar\n"; + } + + #relphot + + { + # relphot only takes lower case gpc1 + my $relphot_camera = lc($camera); + my $command = "$relphot -averages -update"; + $command .= " -D CAMERA $relphot_camera"; + $command .= " -D CATDIR $minidvodb"; + print "$command\n"; + my $mjd_relphot_start = DateTime->now->mjd; # MJD of starting script + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform relphot: $error_code", $minidvodb_id, $error_code); + } + $dtime_relphot = 86400.0*(DateTime->now->mjd - $mjd_relphot_start); # MJD of starting script + print "relphot time $dtime_relphot\n"; + } + + { + + $dtime_script = 86400.0*(DateTime->now->mjd - $mjd_start); + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + #print $command; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to add to minidvodbprocessed: $error_code", $minidvodb_id, $error_code); + } + } +#} else { +# &my_die("dvodb: minidvodb_id = $minidvodb_id not found", $minidvodb_id, $PS_EXIT_UNKNOWN_ERROR); +#} +} else { + print "skipping processing for minidvodb_id = $minidvodb_id\n"; +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $minidbvodb_id = shift; + my $exit_code = shift; # Exit code to add + print STDERR "$msg $minidvodb_id\n"; + +if (defined $minidvodb_id ) { + + my $command = "addtool -minidvodb_id $minidvodb_id"; + $command .= " -addminidvodbprocessed"; + $command .= " -fault $exit_code"; + $command .= " -minidvodb_group $minidvodb_group"; + $command .= " -dtime_relphot $dtime_relphot" if defined $dtime_relphot; + $command .= " -dtime_resort $dtime_resort" if defined $dtime_resort; + $command .= " -dtime_script $dtime_script" if defined $dtime_script; + $command .= " -dbname $dbname" if defined $dbname; + system ($command); + } + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + exit $exit_code; +} + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/nightly_science.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/nightly_science.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/nightly_science.pl (revision 33415) @@ -996,5 +996,10 @@ $cmd .= " -min_num 4"; $cmd .= " -select_good_frac_min 0.05"; - $cmd .= " -set_reduction NIGHTLY_STACK "; + if (defined($reduction)) { + $cmd .= " -set_reduction $reduction "; + } + else { + $cmd .= " -set_reduction NIGHTLY_STACK "; + } $cmd .= " $select "; if ($debug == 1) { @@ -1410,4 +1415,10 @@ } } + if ((defined($science_config{$target}{SELF_WSDIFFS})) && ($science_config{$target}{SELF_WSDIFFS} == 1)) { + foreach my $filter (@filter_list) { + # This one needs to return a state to see if we need to wait on stacking before checking again. + $metadata_out{nsDiffState} = self_WS_diff_queue($date,$target,$filter,$pretend); + } + } } } @@ -1702,30 +1713,87 @@ my ($lunation_start,$lunation_end) = get_lunation_extent($date); - + + +# Get a list of exposures that could be diffed my $new_data_group = "${data_group}.offnight"; my $new_dist_group = "${dist_group}.offnight"; - my $cmd = "$difftool -dbname $dbname -definewarpwarp "; - $cmd .= "-input_label $label -template_label $label "; - $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; - $cmd .= " -simple -set_label $label -filter $filter "; - $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; - - if (defined($pretend)) { - $cmd .= ' -pretend '; - } - if (defined($debug)) { - print STDERR "ON_diffs wants to run this command: $cmd\n"; - } - - if (($debug == 0)&&(!defined($pretend))) { - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run ( command => $cmd, verbose => $verbose ); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); - } - } -} + my $check_cmd = "$difftool -dbname $dbname -definewarpwarp "; + $check_cmd .= "-input_label $label -template_label $label "; + $check_cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; + $check_cmd .= " -mintimediff 40000 "; + $check_cmd .= " -pretend -simple -rerun -set_label $label -filter $filter "; + $check_cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + + my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $check_cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + + # Parse results + my $diffs = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata from difftool -definewarpwarp", 0, 0, $date, $PS_EXIT_PROG_ERROR); + foreach my $diff (@$diffs) { + unless (multi_date_verify_uniqueness_diff($diff->{input_warp_id},$diff->{template_warp_id},$date,$target)) { + # If we don't already have a diff with these inputs, make a diff with these inputs. + my $cmd = "$difftool -dbname $dbname -definewarpwarp "; + $cmd .= "-input_label $label -template_label $label "; + $cmd .= "-warp_id $diff->{input_warp_id} -template_warp_id $diff->{template_warp_id} "; + $cmd .= "-set_workdir $workdir -set_dist_group $new_dist_group -set_data_group $new_data_group "; + $cmd .= " -mintimediff 40000 "; + $cmd .= " -simple -rerun -set_label $label -filter $filter "; + $cmd .= " -dateobs_begin ${lunation_start}T00:00:00 -dateobs_end ${lunation_end}T23:59:59 -distance 1.5 "; + + if (defined($pretend)) { + $cmd .= ' -pretend '; + } + print STDERR "ON_diffs wants to run this command: $cmd\n"; + if (($debug == 0)&&(!defined($pretend))) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $cmd, verbose => $verbose ); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); + } + } + } + } +} + +sub intranight_WS_diff_queue { + my $date = shift; + my $target = shift; + my $filter = shift; + my $pretend = shift; + my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); + my $db = init_gpc_db(); + + my ($stacks_to_finish,$stacks_total) = check_stack_count($label); + if (($stacks_to_finish == 0)&&($stacks_total != 0)) { + my $cmd = "$difftool -dbname $dbname -definewarpstack "; + $cmd .= " -good_frac 0.2 "; + $cmd .= " -warp_label $label -stack_label $label -set_label $label "; + $cmd .= " -set_workdir $workdir -available -set_reduction WARPSTACK -set_dist_group $dist_group "; + $cmd .= " -rerun "; + } +} + +sub check_stack_count { + my $label = shift; + + my $stacks_to_finish = 0; + my $stacks_total = 0; + + my $sth = "SELECT DISTINCT stack_id from stackRun where label = '$label' AND state = 'new'"; + my $data_ref = $db->selectall_arrayref( $sth ); + $stacks_to_finish = $#{ $data_ref } + 1; + + $sth = "SELECT DISTINCT stack_id from stackRun where label = '$label'"; + $data_ref = $db->selectall_arrayref( $sth ); + $stacks_total = $#{ $data_ref } + 1; + return($stacks_to_finish,$stacks_total); +} Index: branches/meh_branches/ppstack_test/ippScripts/scripts/queue_chips_by_night.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/queue_chips_by_night.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/queue_chips_by_night.pl (revision 33415) @@ -0,0 +1,102 @@ +#!/usr/bin/env perl + +use Carp; +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $stdate = `date`; +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use File::Basename; + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my @ARGS = @ARGV; + +my ( $date, $data_group_base, $label, $tess_id, $end_stage, $workdir_base, $dbname ); + +GetOptions( + 'date=s' => \$date, # date, like this 2011-02-12 + 'data_group_base=s' => \$data_group_base, # data_group_base, like this: isp + 'label=s' => \$label, # label + 'dbname|d=s' => \$dbname, # Database name + 'tess_id=s' => \$tess_id, # tess_id + 'end_stage=s' => \$end_stage, # end_stage + 'workdir_base=s' => \$workdir_base, # workdir + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --dbname --label --tess_id --date \n Optional: --data_group_base, end_stage, workdir_base", + -exitval => 3) unless + defined $dbname and + defined $label and + defined $tess_id and + defined $date; + +my $truncdate = $date; + $truncdate =~ s/-//g; +if (defined $data_group_base) { + print "found data_group_base = $data_group_base\n"; +} else { + $data_group_base = $label; + print "using $label as data_group_base\n"; +} +if (defined $end_stage) { + print "found end_stage = $end_stage\n"; +} else { + $end_stage = "camera"; + print "using camera as end_stage\n"; +} +if (defined $workdir_base) { + print "found workdir_base = $workdir_base\n"; +} else { + $workdir_base = "neb://\@HOST\@.0/$dbname/$label"; + print "using $workdir_base as workdir\n"; +} + +# we have all the pieces, now build up the chiptool: + +my $command = "$chiptool -definebyquery -unique"; +$command .= " -dbname $dbname"; +$command .= " -dateobs_begin '$date"."T00:00:00'"; +$command .= " -dateobs_end '$date"."T23:59:59'"; +$command .= " -set_label $label"; +$command .= " -set_data_group $data_group_base.$truncdate"; +$command .= " -set_workdir $workdir_base/$truncdate"; +$command .= " -set_end_stage $end_stage"; +$command .= " -set_tess_id $tess_id"; + +print "\nrunning this command:\n\n$command\n\n"; + + +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform chiptool -definebyquery: $error_code\n"); + exit($error_code); + +} + + Index: branches/meh_branches/ppstack_test/ippScripts/scripts/register_exp.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/register_exp.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/register_exp.pl (revision 33415) @@ -54,5 +54,5 @@ my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $PS_EXIT_CONFIG_ERROR ); -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; @@ -121,5 +121,6 @@ my ($data_group,$dist_group,$chip_workdir,$reduction); ($label,$data_group,$dist_group,$end_stage, - $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-dateobs"), + $tess_id,$chip_workdir,$reduction) = advance_decisions(&value_for_flag($cmdflags,"-exp_name"), + &value_for_flag($cmdflags,"-dateobs"), &value_for_flag($cmdflags,"-exp_type"), &value_for_flag($cmdflags,"-obs_mode"), @@ -170,4 +171,5 @@ ### Pau. sub advance_decisions { + my $exp_name = shift; my $dateobs = shift; my $exptype = shift; @@ -178,4 +180,5 @@ # The strings come pre-quoted. + $exp_name =~ s/\'//g; $dateobs =~ s/\'//g; $exptype =~ s/\'//g; @@ -188,4 +191,9 @@ my $target; + if ($exp_name =~ /^c/) { + print "Skipping because this is a camera commanded exposure: $exp_name\n"; + return(undef,undef,undef,"reg",undef,undef,undef); + } + if ($exptype ne 'OBJECT') { print "Skipping because exptype doesn't claim to be OBJECT: $exptype\n"; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/register_imfile.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/register_imfile.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/register_imfile.pl (revision 33415) @@ -60,5 +60,5 @@ my $ipprc = PS::IPP::Config->new() or my_die_for_add( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die_for_add( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile; pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; @@ -404,4 +404,7 @@ } } + if ($exp_name =~ /a$/) { + return(1); + } return(0); } @@ -517,9 +520,10 @@ carp($msg); - if (defined $exp_id && defined $tmp_class_id and not $no_update) { + if (defined $exp_id && defined $class_id and not $no_update) { my $command = "$regtool -updateprocessedimfile"; $command .= " -exp_id $exp_id"; $command .= " -class_id $class_id"; $command .= " -fault $exit_code"; + $command .= " -set_state pending_burntool "; $command .= " -hostname $host" if defined $host; $command .= " -dbname $dbname" if defined $dbname; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/skycalibration.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/skycalibration.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/skycalibration.pl (revision 33415) @@ -0,0 +1,298 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +## report the program and machine +use Sys::Hostname; +my $host = hostname(); +my $date = `date`; +print "\n\n"; +print "Starting script $0 on $host at $date\n\n"; + +use DateTime; +my $mjd_start = DateTime->now->mjd; # MJD of starting script + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use Data::Dumper; +use File::Temp qw( tempfile ); +use File::Basename; +use PS::IPP::Config 1.01 qw( :standard ); + +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +# Look for programs we need +my $missing_tools; +my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1); +my $psastro = can_run('psastro') or (warn "Can't find psastro" and $missing_tools = 1); +my $fhead = can_run('fhead') or (warn "Can't find fhead" and $missing_tools = 1); +my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +sub HelpMessage { + print "USAGE: skycalibration.pl --skycal_id (skycal_id) --outroot (root) --camera (camera) [options]\n"; + print " --skycal_id (skycal_id) : run identifier\n"; + print " --camera (camera) : Camera name\n"; + print " --dbname (dbname) : Database name\n"; + print " --threads (N) : Number of threads to use\n"; + print " --outroot (root) : Output root name\n"; + print " --path_base (path_base) : path_base of input\n"; + print " --filter (filter) : filter of input\n"; + print " --singlefilter : the input is the result of a single filter analysis"; + print " --reduction (class) : Reduction class\n"; + print " --verbose : Be extra verbose\n"; + print " --no-update : Don't update the database?\n"; + print " --no-op : Don't do any operations?\n"; + print " --redirect-output : send output to the log file,\n"; + print " --save-temps : Save temporary files?\n"; + print " --help : show this message\n"; + exit 2; +} + +my ($skycal_id, $stack_id, $camera, $dbname, $threads, $outroot, $path_base, $filter, $singlefilter, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps); +GetOptions( + 'skycal_id=s' => \$skycal_id, # sky calibration run id + 'stack_id=s' => \$stack_id, # static sky run id + 'camera|c=s' => \$camera, # Camera name + 'dbname|d=s' => \$dbname, # Database name + 'threads=s' => \$threads, # Number of threads to use + 'outroot=s' => \$outroot, # Output root name + 'path_base=s' => \$path_base, # path_base of input + 'filter=s' => \$filter, # filter for input stack + 'reduction=s' => \$reduction, # Reduction class + 'singlefilter' => \$singlefilter, # input was from single filter analysis + 'verbose' => \$verbose, # Print to stdout + 'no-update' => \$no_update, # Don't update the database? + 'no-op' => \$no_op, # Don't do any operations? + 'redirect-output' => \$redirect, + 'save-temps' => \$save_temps, # Save temporary files? + 'help' => sub { HelpMessage() }, +) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; + +pod2usage( + -msg => "Required options: --skycal_id --stack_id --outroot --camera --path_base", + -exitval => 3, + ) unless + defined $skycal_id and + defined $stack_id and + defined $path_base and + defined $outroot and + defined $camera; + +my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $skycal_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration + +my $logDest = $ipprc->filename("LOG.EXP", $outroot); +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $skycal_id, $PS_EXIT_SYS_ERROR ) if $redirect; + +my $traceDest = $ipprc->filename("TRACE.EXP", $outroot); +my $stats = $ipprc->filename("PSASTRO.STATS", $outroot); + +my $outbase = basename($outroot); + +# Get list of input images to stack photometry +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files +my $files; + +# Recipes to use based on reduction class +$reduction = 'DEFAULT' unless defined $reduction; + +# generate the input +my $cmdflags = ""; + +my $recipe_psastro = $ipprc->reduction($reduction, 'STATICSKY_CALIBRATION'); # Recipe to use for psastro +unless ($recipe_psastro) { + &my_die("Couldn't find selected reduction for STATICSKY_CALIBRATION: $reduction\n", $skycal_id, $PS_EXIT_CONFIG_ERROR); +} + +# XXX: We are making the bad assumption here that the staticksky run used the same reduction class +# as this skycalRun +my $recipe_psphot = $ipprc->reduction($reduction, "STACKPHOT_PSPHOT"); +unless ($recipe_psphot) { + &my_die("Couldn't find selected reduction for STATCKPHOT_PSPHOT: $reduction\n", $skycal_id, $PS_EXIT_CONFIG_ERROR); +} + +print "reduction: $reduction\n"; +print "recipe_psastro: $recipe_psastro\n"; +print "recipe_psphot: $recipe_psphot\n"; + +my $configuration = $ipprc->filename("PSASTRO.CONFIG", $outroot); + +# find input file +# First check the expected file name where the stack_id is the FILE_ID +my $file; +if (!$singlefilter) { + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id); + if (! $ipprc->file_resolve($file)) { + # XXX: Beginning of section that can be removed eventually + + # no file with the expected name found + # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID + # but instead used FILE_ID = [0 .. num_filters-1] + print "\nfailed to resolve $file\n"; + if (!$filter) { + &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + } + print "Trying old style FILE_ID\n"; + my $max_filters = 5; + for (my $i=0; $i < $max_filters; $i++) { + my $file_id = sprintf "%03d", $i; + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id); + my $resolved = $ipprc->file_resolve($file); + if (!$resolved) { + # we fail here assumming that if file_id N doesn't exist, neither to N+1 + print "\nfailed to resolve $file\n\n"; + &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Check the filter id for this file + my $command = "$fhead $resolved | grep FILTERID"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name) + my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf); + my $this_filter = substr $filt, 1, 7; + + # if it matches we're done + last if $this_filter eq $filter; + + # nope loop around to try the next one + print "Input file for $filter is not $file ($this_filter)\n"; + $file = undef; + } + print "\nInput file for $stack_id filter: $filter: $file\n"; + + # XXX: End of section that can be removed eventually + } +} else { + # input is from a single filter static sky run use a different file rule + # XXX: can't we just make staticsky.pl use the same file rule? + $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); + &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($file); +} + +if (!$file) { + &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR); +} + +# Perform psastro analysis +my $output_sources_filerule = 'PSASTRO.OUTPUT.CMF'; +{ + my $command = "$psastro $outroot"; + $command .= " -file $file"; +# $command .= " -threads $threads" if defined $threads; # psastro doesn't currently accept -threads + $command .= " -recipe PSASTRO $recipe_psastro"; + $command .= " -recipe PSPHOT $recipe_psphot"; + $command .= " -F PSASTRO.OUTPUT $output_sources_filerule"; + $command .= " -stats $stats"; + $command .= " -recipe PPSTATS CAMSTATS"; + $command .= " -dumpconfig $configuration" if $configuration; + $command .= " -tracedest $traceDest"; + + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $skycal_id, $error_code); + } + + &my_die("Couldn't find stats file: $stats", $skycal_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($stats); + + { + my $resolved = $ipprc->file_resolve($stats) + or &my_die("failed to resolve stats file: $stats", $skycal_id, $PS_EXIT_SYS_ERROR) ; + my $command = "$ppStatsFromMetadata $resolved - STATICSKY_CAL"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform psastro: $error_code", $skycal_id, $error_code); + } + $cmdflags = join "", @$stdout_buf; + chomp $cmdflags; + } + + my $quality; + ($quality) = $cmdflags =~ /-quality (\d+)/; + if (!$quality) { + my $outputSources = $ipprc->filename($output_sources_filerule, $outroot); + &my_die("Couldn't find expected output file: $outputSources", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($outputSources); + + &my_die("Couldn't find expected output file: $configuration", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($configuration); + } + } else { + print "Not executing: $command\n"; + } +} + + +# Add the result to the database +{ + my $command = "$staticskytool -skycal_id $skycal_id"; + $command .= " -addskycalresult -path_base $outroot"; + $command .= " $cmdflags"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -dbname $dbname" if defined $dbname; + + unless($no_update) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform staticskytool -addresult"; + warn($err_message); + exit $error_code; + } + } else { + print "skipping $command\n"; + } +} + +exit 0; + + +sub my_die +{ + my $msg = shift; # Warning message on die + my $skycal_id = shift; # Diff identifier + my $exit_code = shift; # Exit code to add + + $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code; + + warn($msg); + if (defined $skycal_id and not $no_update) { + my $command = "$staticskytool -skycal_id $skycal_id -fault $exit_code"; + $command .= " -addskycalresult"; + $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); + $command .= " -hostname $host" if defined $host; + $command .= " -path_base $outroot" if defined $outroot; + $command .= " -dbname $dbname" if defined $dbname; + run(command => $command, verbose => $verbose); + } + exit $exit_code; +} + +END { + my $exit = $?; + system("sync") == 0 or die "failed to execute sync: $!"; + $? = $exit; +} + +__END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/stack_skycell.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/stack_skycell.pl (revision 33415) @@ -104,5 +104,5 @@ } -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect; +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect; my $neb; @@ -243,4 +243,5 @@ print $listFile "END\n\n"; } +close($listFile); print "I've checked everything: $stack_id\n"; @@ -491,4 +492,17 @@ &my_die("Couldn't find expected output file: $file", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $stack_id, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $stack_id, $PS_EXIT_SYS_ERROR); + } + } + ##### + if ($replicate and $neb) { $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n", $stack_id, $PS_EXIT_SYS_ERROR); Index: branches/meh_branches/ppstack_test/ippScripts/scripts/videophot_process.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/videophot_process.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/videophot_process.pl (revision 33415) @@ -8,7 +8,4 @@ my $host = hostname(); my $date = `date`; -print "\n\n"; -print "Starting script $0 on $host at $date\n\n"; - use DateTime; my $mjd_start = DateTime->now->mjd; # MJD of starting script @@ -25,24 +22,34 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); use Pod::Usage qw( pod2usage ); +use File::Temp qw( tempfile ); +use File::Basename qw( basename ); # Look for programs we need my $missing_tools; my $vptool = can_run('vptool') or (warn "Can't find vptool" and $missing_tools = 1); -my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); my $psvideophot = can_run('psvideophot') or (warn "Can't find psvideophot" and $missing_tools = 1); +my $dumpvideo = can_run('dumpvideo') or (warn "Can't find dumpvideo" and $missing_tools = 1); my $listvideocells = can_run('listvideocells.pl') or (warn "Can't find listvideocells.pl" and $missing_tools = 1); +my $dsreg = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1); +my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1); if ($missing_tools) { warn("Can't find required tools."); exit($PS_EXIT_CONFIG_ERROR); } -my ($vp_id, $exp_id, $camera, $outroot, $dbname, $verbose, $no_update, $no_op, $redirect); +my ($vp_id, $camera, $outroot, $dest_id, $dbname, $verbose, $no_update, $no_op, $redirect, $save_temps); +my ($product, $save_video_cube, $ds_dbname, $ds_dbhost); GetOptions( 'vp_id=s' => \$vp_id, - 'exp_id=s' => \$exp_id, 'camera=s' => \$camera, 'outroot=s' => \$outroot, + 'dest_id=s' => \$dest_id, + 'product=s' => \$product, + 'save-video-cube' => \$save_video_cube, + 'ds_dbname=s' => \$ds_dbname, + 'ds_dbhost=s' => \$ds_dbhost, 'dbname|d=s' => \$dbname, 'verbose' => \$verbose, + 'save-temps' => \$save_temps, 'no-update' => \$no_update, # Don't update the database? 'no-op' => \$no_op, # Don't do any operations? @@ -52,41 +59,50 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; pod2usage( - -msg => "Required options: --vp_id --exp_id --outroot --camera", + -msg => "Required options: --vp_id --outroot --camera", -exitval => 3, ) unless defined $vp_id - and defined $exp_id and defined $camera and defined $outroot; +pod2usage( + -msg => " --product -ds_dbname and --dsdbhost are required if --dest_id", + -exitval => 3, + ) if ($dest_id and !( defined $product and defined $ds_dbname and defined $ds_dbhost)); + $no_update = 1 if $no_op; - -my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, 'nochip', 'nocell', $PS_EXIT_CONFIG_ERROR ); # IPP configuration +$vptool .= " -dbname $dbname" if $dbname; + +my $ipprc = PS::IPP::Config->new($camera) or my_die( "Unable to set up", $vp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration my $logDest = $ipprc->filename("LOG.EXP", $outroot); -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $vp_id, 'nochip', 'nocell', $PS_EXIT_SYS_ERROR ) if $redirect; - -my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF); - -# Get list of components for subtraction +if ($redirect) { + $ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $vp_id, $PS_EXIT_SYS_ERROR ); + + print "\n\n"; + print "Starting script $0 on $host at $date\n\n"; +} + +# Get list of chips with pending video cells my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files my $files; { -# my $command = "$vptool -pendingimfile -vp_id $vp_id"; -# $command .= " -dbname $dbname" if defined $dbname; - my $command = "$regtool -processedimfile -video_cells -exp_id $exp_id"; - $command .= " -dbname $dbname" if defined $dbname; + my $command = "$vptool -pendingimfile -vp_id $vp_id"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, 'nochip', 'nocell', $error_code); - } - - my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR); - $files = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", $vp_id, 'nochip', 'nocell', $PS_EXIT_PROG_ERROR); -} + &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code); + } + + my $output = join "", @$stdout_buf; + if ($output) { + my $metadata = $mdcParser->parse($output) or + &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR); + $files = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR); + } +} + foreach my $file (@$files) { @@ -95,5 +111,5 @@ my $resolved = $ipprc->file_resolve($uri); - &my_die("Unable to resolve $uri", $vp_id, $class_id, 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless $resolved; + &my_die("Unable to resolve $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless $resolved; my $command = "$listvideocells --file $resolved"; @@ -101,18 +117,22 @@ run(command => $command, verbose => $verbose); unless ($success) { + check_input_file($uri, $vp_id, $class_id, 'nocell'); $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $vp_id, $class_id, - 'nocell', $error_code); + &my_die("Unable to perform $command: $error_code", $vp_id, $error_code); } my @video_cells = split "\n", (join "", @$stdout_buf); - &my_die("No video cells found in $uri", $vp_id, $class_id, - 'nocell', $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells; + &my_die("No video cells found in $uri", $vp_id, $PS_EXIT_UNKNOWN_ERROR) unless scalar @video_cells; foreach my $cell_id (@video_cells) { - my $command = "$psvideophot $outroot"; + my $path_base = "$outroot.$class_id.$cell_id"; + my $error; + my $output = $ipprc->prepare_output("PSVIDEOPHOT.OUTPUT", $path_base, undef, 1, \$error) + or &my_die("failed to prepare output file for PSVIDEOPHOT.OUTPUT", $vp_id, $error); + my $command = "$psvideophot $output"; $command .= " -file $uri"; + $command .= " -class_id $class_id"; $command .= " -cell_id $cell_id"; - $command .= " -dbname $dbname" if defined $dbname; + my $vpstart = DateTime->now->mjd; unless ($no_op) { my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -120,16 +140,31 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform ppSub: $error_code", $vp_id, $class_id, $cell_id, $error_code); + &my_die("Unable to perform psvideophot: $error_code", $vp_id, $error_code); } - - # check_output($outputStats, 1); } else { print "Not executing: $command\n"; } + # dump the video cell and the video table to a file + if ($save_video_cube) { + my $output = $path_base . ".fits"; + my $extname = "$cell_id"; + my $command = "$dumpvideo $uri $extname $output -includetable"; + unless ($no_op) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dumpvideo: $error_code", $vp_id, $error_code); + } + } else { + print "Not executing: $command\n"; + } + } + unless ($no_update) { - my $command = "$vptool -addvpcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; - $command .= " -path_base $outroot"; - - $command .= " -dbname $dbname" if defined $dbname; + my $command = "$vptool -addprocessedcell -vp_id $vp_id -class_id $class_id -cell_id $cell_id"; + $command .= " -path_base $path_base"; + $command .= " -hostname $host"; + $command .= " -dtime_photom " . ((DateTime->now->mjd - $vpstart) * 86400); my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -138,18 +173,113 @@ $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); my $err_message = "Unable to perform vptool -addvpcell"; - &my_die("$err_message: $error_code", $vp_id, $class_id, $cell_id, $error_code); + &my_die("$err_message: $error_code", $vp_id, $error_code); } } } } +if ($dest_id) { + my $command = "$vptool -processedcell -vp_id $vp_id"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform vptool -pendingimfile: $error_code", $vp_id, $error_code); + } + + my $output = join "", @$stdout_buf; + my $cells; + if ($output) { + my $metadata = $mdcParser->parse($output) or + &my_die("Unable to parse metadata config doc", $vp_id, $PS_EXIT_PROG_ERROR); + $cells = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", $vp_id, $PS_EXIT_PROG_ERROR); + } + my ($reglist, $reglistName) = tempfile("/tmp/filelist.$vp_id.XXXX", UNLINK => !$save_temps); + + if ($redirect) { + print $reglist "$logDest|||text|\n"; + } + # list of extensions and types for files to distribute + my @exts = qw( vpt ); + my @types = qw( table ); + if ($save_video_cube) { + push @exts, 'fits'; + push @types, 'fits'; + } + foreach my $cell (@$cells) { + my $path_base = $cell->{path_base}; + for (my $i = 0; $i < scalar @exts; $i++) { + my $file = "$path_base.$exts[$i]"; + my $type = $types[$i]; + + print $reglist "$file|||$type|\n"; + } + } + close $reglist or &my_die("failed to close $reglistName", $vp_id, $PS_EXIT_UNKNOWN_ERROR); + unless ($no_update or $no_op) { + my $fileset = basename($outroot); + my $command = "$dsreg --add $fileset --product $product --type dump --list $reglistName"; + $command .= " --abspath --link"; + $command .= " --dbname $ds_dbname --dbhost $ds_dbhost"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command error_code: $error_code", $vp_id, $error_code); + } + } +} + +unless ($no_update) { + my $command = "$vptool -updaterun -vp_id $vp_id -set_state full"; + $command .= " -set_outroot $outroot"; + $command .= " -set_hostname $host"; + $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform vptool -updaterun"; + &my_die("$err_message: $error_code", $vp_id, $error_code); + } +} exit 0; + +sub check_input_file { + my $uri = shift; + my $vp_id = shift; + my $class_id = shift; + my $cell_id = shift; + my $resolved = $ipprc->file_resolve($uri); + + my $tryrepair = 0; + if (!-e $resolved) { + printf STDERR "instance $resolved for $uri does not exist\n"; + $tryrepair = 1; + } elsif (-s $resolved == 0) { + printf STDERR "instance $resolved for $uri is empty\n"; + $tryrepair = 1; + } + if ($tryrepair) { + my $scheme = file_scheme($uri); + if ($scheme and ($scheme = 'neb')) { + my $command = "$nebrepair $uri"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + my $err_message = "Unable to perform nebrepair"; + &my_die("$err_message: $error_code", $vp_id, $error_code); + } + } + } +} sub my_die { my $msg = shift; # Warning message on die - my $vp_id = shift; - my $class_id = shift; # chip identifier - my $cell_id = shift; # chip identifier + my $vp_id = shift; # vpRun id my $exit_code = shift; # Exit code to add @@ -157,11 +287,12 @@ warn($msg); - if (defined $vp_id and defined $class_id and not $no_update) { - my $command = "$vptool -vp_id $vp_id -class_id $class_id -cell_id -fault $exit_code"; - $command .= " -addvpcell"; - $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400)); - $command .= " -hostname $host" if defined $host; - $command .= " -path_base $outroot" if defined $outroot; - $command .= " -dbname $dbname" if defined $dbname; + if (defined $vp_id and not $no_update) { + my $command = "$vptool -vp_id $vp_id"; + $command .= " -updaterun"; + $command .= " -set_fault $exit_code"; + $command .= " -set_hostname $host" if defined $host; + $command .= " -set_outroot $outroot" if defined $outroot; + $command .= " -set_dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400); + run(command => $command, verbose => $verbose); } @@ -169,9 +300,3 @@ } -END { - my $exit = $?; - system("sync") == 0 or die "failed to execute sync: $!"; - $? = $exit; -} - __END__ Index: branches/meh_branches/ppstack_test/ippScripts/scripts/warp_overlap.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/warp_overlap.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/warp_overlap.pl (revision 33415) @@ -57,5 +57,5 @@ my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration -$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile; +$ipprc->redirect_to_logfile($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile; &my_die("Tessellation identifier not provided: $tess_dir", $warp_id, $PS_EXIT_SYS_ERROR) unless $tess_dir ne "NULL"; Index: branches/meh_branches/ppstack_test/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/warp_skycell.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/warp_skycell.pl (revision 33415) @@ -38,5 +38,4 @@ my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps); -my $zaplog = 0; GetOptions( @@ -55,5 +54,4 @@ 'no-update' => \$no_update, # Don't update the database? 'no-op' => \$no_op, # Don't do any operations? - 'zaplog' => \$zaplog, # delete previous logfile 'redirect-output' => \$redirect, 'save-temps' => \$save_temps, # Save temporary files? @@ -77,9 +75,9 @@ my $do_stats; if ($run_state eq 'new') { - $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, $zaplog); + $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id); $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1); $do_stats = 1; } elsif ($run_state eq 'update') { - $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 1); + $logDest = $ipprc->filename("LOG.EXP.UPDATE", $outroot, $skycell_id); $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1); } else { @@ -93,5 +91,5 @@ } -$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; +$ipprc->redirect_to_logfile($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect; # Recipes to use based on reduction class @@ -333,5 +331,5 @@ } } - + print "Quality: $quality\n"; unless ($no_update) { # XXX change -tess_id to -tess_dir when db is updated @@ -358,18 +356,42 @@ } } else { + + my $quality = 0; + + unless ($ipprc->file_exists($outputPSF)) { + $quality = 8006; # bad data. + } +# if (!$quality) { +# check_output($outputImage, 0); +# check_output($outputMask, 0); +# check_output($outputWeight, 0); +# check_output($outputSources, 1); +# check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF') ; +# } + print "Quality: $quality\n"; + # $run_state eq 'update' unless ($no_update) { - my $command = "$warptool -tofullskyfile"; - $command .= " -warp_id $warp_id"; - $command .= " -skycell_id $skycell_id"; - $command .= " -set_magicked $magicked" if $magicked; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - warn("Unable to perform warptool -addwarped: $error_code\n"); - exit($error_code); + my $try = 0; + while (1) { + $try++; + my $command = "$warptool -tofullskyfile"; + $command .= " -warp_id $warp_id"; + $command .= " -skycell_id $skycell_id"; + $command .= " -set_magicked $magicked" if $magicked; + $command .= " -set_quality $quality" if $quality; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + warn("Unable to perform warptool -addwarped: $error_code\n"); + exit($error_code) if $try >= 2; + sleep 10; + warn("Trying again\n"); + } else { + last; + } } } @@ -407,4 +429,17 @@ &my_die("Couldn't find expected output file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file); + + # Funpack to confirm we've really made things correctly + my $diskfile = $ipprc->file_resolve($file); + if ($diskfile =~ /fits/) { + my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + my $check_command = "$funpack -S $diskfile > /dev/null"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $check_command, verbose => $verbose); + if (!$success) { + &my_die("Output file not a valid fits file: $file", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR); + } + } + ##### if ($replicate and $neb) { Index: branches/meh_branches/ppstack_test/ippScripts/scripts/whichimage =================================================================== --- branches/meh_branches/ppstack_test/ippScripts/scripts/whichimage (revision 32161) +++ branches/meh_branches/ppstack_test/ippScripts/scripts/whichimage (revision 33415) @@ -60,5 +60,5 @@ my $sign = 1; - if ($h < 0) { + if ($h =~ /-/) { $sign = -1; $h = - $h; @@ -76,5 +76,5 @@ my $sign = 1; - if ($d < 0) { + if ($d =~ /-/) { $sign = -1; $d = - $d; Index: branches/meh_branches/ppstack_test/ippTasks/Makefile.am =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/Makefile.am (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/Makefile.am (revision 33415) @@ -21,4 +21,5 @@ destreak.cleanup.pro \ diff.pro \ + skycal.pro \ stack.pro \ staticsky.pro \ @@ -37,4 +38,5 @@ minidvodb.pro \ minidvodbcopy.pro \ + mergedvodb.pro \ nightly_science.pro \ burntool.pro \ @@ -42,5 +44,6 @@ background.pro \ diffphot.pro \ - lap.pro + lap.pro \ + vp.pro other_files = \ Index: branches/meh_branches/ppstack_test/ippTasks/addstar.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/addstar.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/addstar.pro (revision 33415) @@ -26,4 +26,7 @@ active true end + task addstar.exp.load.staticsky_multi + active true + end task addstar.exp.run active true @@ -41,4 +44,7 @@ active false end + task addstar.exp.load.staticsky_multi + active false + end task addstar.exp.run active false @@ -47,10 +53,28 @@ macro addstar.revert.off - task addstar.revert + task addstar.revert.cam + active false + end + task addstar.revert.stack + active false + end + task addstar.revert.staticsky + active false + end + task addstar.revert.staticsky_multi active false end end macro addstar.revert.on - task addstar.revert + task addstar.revert.cam + active true + end + task addstar.revert.stack + active true + end + task addstar.revert.staticsky_multi + active true + end + task addstar.revert.staticsky active true end @@ -59,6 +83,10 @@ # this variable will cycle through the known database names -$addstar_DB = 0 -$addstar_revert_DB = 0 +$addstar_DB = 0 +#this may not work for more databases (addstar) will do that later) +$addstar_revert_DB_C = 0 +$addstar_revert_DB_S = 0 +$addstar_revert_DB_SS = 0 +$addstar_revert_DB_SSM = 0 @@ -175,4 +203,5 @@ end end + task addstar.exp.load.staticsky host local @@ -229,4 +258,61 @@ end end + +task addstar.exp.load.staticsky_multi + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/addstar.exp.log + + task.exec + # if ($LABEL:n == 0) break + $run = addtool -pendingexp -stage staticsky_multi + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_DB + $run = $run -dbname $DB:$addstar_DB + $addstar_DB ++ + if ($addstar_DB >= $DB:n) set addstar_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout addPendingExp -key add_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook addPendingExp + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup addPendingExp + end + + # default exit status + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + # run the addstar script on pending exposures task addstar.exp.run @@ -252,6 +338,8 @@ book getword addPendingExp $pageName exp_tag -var EXP_TAG book getword addPendingExp $pageName add_id -var ADD_ID + book getword addPendingExp $pageName stage_extra1 -var STAGE_EXTRA1 book getword addPendingExp $pageName stageroot -var STAGEROOT - book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage -var STAGE + book getword addPendingExp $pageName stage_id -var STAGE_ID book getword addPendingExp $pageName workdir -var WORKDIR_TEMPLATE book getword addPendingExp $pageName reduction -var REDUCTION @@ -285,4 +373,7 @@ sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID end + if ("$STAGE" == "staticsky_multi") + sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID + end if ("$STAGE" == "stack") sprintf outroot "%s.add.%s" $STAGEROOT $ADD_ID @@ -296,4 +387,10 @@ if ("$REDUCTION" != "NULL") $run = $run --reduction $REDUCTION + end + if ("$STAGE" == "staticsky_multi") + $run = $run --multi_num $STAGE_EXTRA1 + end + if ("$STAGE" == "cam") + $run = $run --stage_id $STAGE_ID end if ("$IMAGE_ONLY" == "T") @@ -340,5 +437,5 @@ end -task addstar.revert +task addstar.revert.cam host local @@ -354,13 +451,13 @@ task.exec if ($LABEL:n == 0) break - $run = addtool -revertprocessedexp - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$addstar_revert_DB - $run = $run -dbname $DB:$addstar_revert_DB - $addstar_revert_DB ++ - if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0 + $run = addtool -revertprocessedexp -stage cam + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_C + $run = $run -dbname $DB:$addstar_revert_DB_C + $addstar_revert_DB_C ++ + if ($addstar_revert_DB_C >= $DB:n) set addstar_revert_DB_C = 0 end add_poll_labels run @@ -386,2 +483,140 @@ end end +task addstar.revert.stack + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage stack + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_S + $run = $run -dbname $DB:$addstar_revert_DB_S + $addstar_revert_DB_S ++ + if ($addstar_revert_DB_S >= $DB:n) set addstar_revert_DB_S = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task addstar.revert.staticsky + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage staticsky + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SS + $run = $run -dbname $DB:$addstar_revert_DB_SS + $addstar_revert_DB_SS ++ + if ($addstar_revert_DB_SS >= $DB:n) set addstar_revert_DB_SS = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end +task addstar.revert.staticsky_multi + host local + + periods -poll 5.0 + periods -exec 60.0 + periods -timeout 1200 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + $run = addtool -revertprocessedexp -stage staticsky_multi + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$addstar_revert_DB_SSM + $run = $run -dbname $DB:$addstar_revert_DB_SSM + $addstar_revert_DB_SSM ++ + if ($addstar_revert_DB_SSM >= $DB:n) set addstar_revert_DB_SSM = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/meh_branches/ppstack_test/ippTasks/destreak.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/destreak.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/destreak.pro (revision 33415) @@ -206,5 +206,6 @@ book getword magicToDS $pageName magicked -var MAGICKED book getword magicToDS $pageName dbname -var DBNAME - + book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID + book getword magicToDS $pageName mismatched_tess -var MISMATCHED_TESS substr $COMPONENT 0 3 COMP_HEAD @@ -224,4 +225,8 @@ $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $WORKDIR --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED --run-state $RUN_STATE + if ($MISMATCHED_TESS) + book getword magicToDS $pageName diff_tess_id -var DIFF_TESS_ID + $run = $run --mismatched_tess --diff_tess_id $DIFF_TESS_ID + end add_standard_args run Index: branches/meh_branches/ppstack_test/ippTasks/diskbalance.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/diskbalance.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/diskbalance.pro (revision 33415) @@ -15,4 +15,8 @@ book init balanceControl + +book init targetPending +book init targetControl + macro balance.reset book init balancePending @@ -41,4 +45,30 @@ end +macro target.reset + book init targetPending +end + +macro target.status + book listbook targetPending +end + +macro target.on + task target.load + active true + end + task target.run + active true + end +end + +macro target.off + task target.load + active false + end + task target.run + active false + end +end + macro set.host.for.balance if ($0 != 2) @@ -71,4 +101,15 @@ book setword balanceControl control mDestinations $2 end + +macro set.balance.range + if ($0 != 3) + echo "USAGE: set.balance.range (SO_ID_START) (SO_ID_RANGE)" + break + end + + $BALANCE_SO_ID_START = $1 + $BALANCE_SO_ID_RANGE = $2 +end + macro show.balance.numbers @@ -92,4 +133,25 @@ end +macro set.target.range + if ($0 != 3) + echo "USAGE: set.target.range (SO_ID_START) (SO_ID_RANGE)" + break + end + + $TARGET_SO_ID_START = $1 + $TARGET_SO_ID_RANGE = $2 +end + + +macro show.target.numbers + echo "target so_ids: $TARGET_SO_ID_START $TARGET_SO_ID_RANGE" +end + +macro show.targetPending + book npages targetPending -var N + echo "$N pages in book" + book listbook targetPending +end + # the balance process interacts with only the single Nebulous server @@ -104,5 +166,5 @@ $BALANCE_LIMIT = 500 $BALANCE_OFFSET = 0 -$BALANCE_TARGET = "SOURCE" +$BALANCE_TARGET = SOURCE # Select Nebulous objects which should be shuffled task balance.load @@ -250,5 +312,155 @@ - - - +# This value will need to be incremented periodically. It looks like we grow by about 5e7 so_ids per month. +$TARGET_BIG_SO_ID = 6e8 +$TARGET_SO_ID_START = int($TARGET_BIG_SO_ID * rnd(0)) +$TARGET_SO_ID_RANGE = 500000 +$TARGET_LIMIT = 500 +$TARGET_OFFSET = 0 +$TARGET_TARGET = SOURCE +# Select Nebulous objects which should be shuffled +task target.load + host local + + periods -poll 60 + periods -exec 60 + periods -timeout 1500 + npending 1 + + # logs + stdout NULL + stderr $LOGSUBDIR/target.log + + task.exec + book npages targetPending -var N + if ($N > 2000) + process_cleanup targetPending + break + end + + command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingtarget --limit $TARGET_LIMIT --offset $TARGET_OFFSET --so_id_start $TARGET_SO_ID_START --so_id_range $TARGET_SO_ID_RANGE + end + + # success : 0 -- we did not hit the limit, advance so_id counter + task.exit 0 + # advance the so_id counter + $TARGET_SO_ID_START = $TARGET_SO_ID_START + $TARGET_SO_ID_RANGE + $TARGET_OFFSET = 0 + # convert 'stdout' to book format + ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT + + if ($VERBOSE > 2) + book listbook targetPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup targetPending + end + + # success : 1 -- we DID hit the limit, do NOT advance so_id counter + task.exit 1 + # convert 'stdout' to book format + ipptool2book stdout targetPending -key key -uniq -setword pantaskState INIT + $TARGET_OFFSET = $TARGET_OFFSET + $TARGET_LIMIT + if ($VERBOSE > 2) + book listbook targetPending + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup targetPending + end + + # out of so_id range, reset + task.exit 10 + # advance the so_id counter + $TARGET_SO_ID_START = 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# task to execute shuffles +task target.run + periods -poll 0.5 + periods -exec 5 + periods -timeout 5 + npending 200 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 5 + + book npages targetPending -var N + if ($NETWORK == 0) break + if ($N == 0) break + + # look for new objects in targetPending + book getpage targetPending 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword targetPending $pageName pantaskState RUN + + book getword targetPending $pageName key -var KEY + book getword targetPending $pageName source_name -var SOURCE + book getword targetPending $pageName source_host -var SOURCE_HOST + book getword targetPending $pageName destination_name -var DESTINATION + book getword targetPending $pageName destination_host -var DEST_HOST + book getword targetPending $pageName destination_uri -var DEST_URI + book getword targetPending $pageName ins_id -var INS_ID + + # Fix this with multihost restriction when possible. + if ("$TARGET_TARGET" == "DESTINATION") + set.host.for.balance $DEST_HOST + else + set.host.for.balance $SOURCE_HOST + end + + + stdout NULL + stderr $LOGSUBDIR/target.log + + $run = neb-insedit --key $KEY --ins_id $INS_ID --uri $DEST_URI --volume $DESTINATION + $run = $run --db $NEB_DB --host $NEB_HOST --user $NEB_USER --pass $NEB_PASS + $run = $run --neb_host http://ippc04/nebulous + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + periods -exec 0.05 + command $run + end + + # default exit status + task.exit default + process_exit targetPending $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword targetPending $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword targetPending $options:0 pantaskState TIMEOUT + end +end + + Index: branches/meh_branches/ppstack_test/ippTasks/fake.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/fake.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/fake.pro (revision 33415) @@ -146,5 +146,7 @@ # specify choice of local or remote host based on camera and fake (class_id) - set.host.for.camera $CAMERA $CLASS_ID + # XXX: this is not worth the trouble. use anyhost + # set.host.for.camera $CAMERA $CLASS_ID + host anyhost # set the WORKDIR variable Index: branches/meh_branches/ppstack_test/ippTasks/ipphosts.mhpcc.config =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/ipphosts.mhpcc.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/ipphosts.mhpcc.config (revision 33415) @@ -6,135 +6,151 @@ # randomly. the only exception is ipp004, which is the DVO db host -# NOTE: wave 1 machines have 10TB; wave 2 & 3 machines have 20TB -# for balance, use 1 host per 10 TB in the partition +# I've allocated hosts based on disk size. +# 40TB => 2 units of skycells +# 20TB => 1 unit of skycells +# Each host gets one OTA/chip of data. +# These hosts are skipped: +# ipp004 : dvo +# ipp005 : dvo +# ipp026 : crashes +# ipp029 : crashes +# ipp045 : reference cat ipphosts METADATA camera STR skycell - count S32 76 - sky00 STR ipp005 + count S32 86 + sky00 STR ipp006 sky01 STR ipp006 sky02 STR ipp007 - sky03 STR ipp008 - sky04 STR ipp009 - sky05 STR ipp010 - sky06 STR ipp011 - sky07 STR ipp012 - sky08 STR ipp012 - sky09 STR ipp013 - sky10 STR ipp014 - sky11 STR ipp014 - sky12 STR ipp015 - sky13 STR ipp016 - sky14 STR ipp016 - sky15 STR ipp017 - sky16 STR ipp018 - sky17 STR ipp019 - sky18 STR ipp020 - sky19 STR ipp021 - sky20 STR ipp023 - sky21 STR ipp024 - sky22 STR ipp025 - sky23 STR ipp026 - sky24 STR ipp027 - sky25 STR ipp027 - sky26 STR ipp028 - sky27 STR ipp028 - sky28 STR ipp029 - sky29 STR ipp029 - sky30 STR ipp030 - sky31 STR ipp030 - sky32 STR ipp031 - sky33 STR ipp031 - sky34 STR ipp032 - sky35 STR ipp032 - sky36 STR ipp033 - sky37 STR ipp033 - sky38 STR ipp034 - sky39 STR ipp034 - sky40 STR ipp035 - sky41 STR ipp035 - sky42 STR ipp036 + sky03 STR ipp007 + sky04 STR ipp008 + sky05 STR ipp008 + sky06 STR ipp009 + sky07 STR ipp009 + sky08 STR ipp010 + sky09 STR ipp010 + sky10 STR ipp011 + sky11 STR ipp011 + sky12 STR ipp012 + sky13 STR ipp012 + sky14 STR ipp013 + sky15 STR ipp013 + sky16 STR ipp014 + sky17 STR ipp014 + sky18 STR ipp015 + sky19 STR ipp015 + sky20 STR ipp016 + sky21 STR ipp016 + sky22 STR ipp017 + sky23 STR ipp017 + sky24 STR ipp018 + sky25 STR ipp018 + sky26 STR ipp019 + sky27 STR ipp019 + sky28 STR ipp020 + sky29 STR ipp020 + sky30 STR ipp021 + sky31 STR ipp021 + sky32 STR ipp023 + sky33 STR ipp024 + sky34 STR ipp025 + sky35 STR ipp027 + sky36 STR ipp028 + sky37 STR ipp030 + sky38 STR ipp031 + sky39 STR ipp032 + sky40 STR ipp033 + sky41 STR ipp034 + sky42 STR ipp035 sky43 STR ipp036 sky44 STR ipp037 - sky45 STR ipp037 - sky46 STR ipp038 - sky47 STR ipp038 - sky48 STR ipp039 - sky49 STR ipp039 - sky50 STR ipp040 - sky51 STR ipp040 - sky52 STR ipp041 - sky53 STR ipp041 - sky54 STR ipp042 - sky55 STR ipp042 - sky56 STR ipp043 - sky57 STR ipp043 - sky58 STR ipp044 - sky59 STR ipp044 - sky60 STR ipp046 - sky61 STR ipp046 - sky62 STR ipp047 - sky63 STR ipp047 - sky64 STR ipp048 - sky65 STR ipp048 - sky66 STR ipp049 - sky67 STR ipp049 - sky68 STR ipp050 - sky69 STR ipp050 - sky70 STR ipp051 - sky71 STR ipp051 - sky72 STR ipp052 - sky73 STR ipp052 - sky74 STR ipp053 - sky75 STR ipp053 + sky45 STR ipp038 + sky46 STR ipp039 + sky47 STR ipp040 + sky48 STR ipp041 + sky49 STR ipp042 + sky50 STR ipp043 + sky51 STR ipp044 + sky52 STR ipp046 + sky53 STR ipp047 + sky54 STR ipp048 + sky55 STR ipp049 + sky56 STR ipp050 + sky57 STR ipp051 + sky58 STR ipp052 + sky59 STR ipp053 + sky60 STR ipp054 + sky61 STR ipp054 + sky62 STR ipp055 + sky63 STR ipp055 + sky64 STR ipp056 + sky65 STR ipp056 + sky66 STR ipp057 + sky67 STR ipp057 + sky68 STR ipp058 + sky69 STR ipp058 + sky70 STR ipp059 + sky71 STR ipp059 + sky72 STR ipp060 + sky73 STR ipp060 + sky74 STR ipp061 + sky75 STR ipp061 + sky76 STR ipp062 + sky77 STR ipp062 + sky78 STR ipp063 + sky79 STR ipp063 + sky80 STR ipp064 + sky81 STR ipp064 + sky82 STR ipp065 + sky83 STR ipp065 + sky84 STR ipp066 + sky85 STR ipp066 END -# we have 18 10TB nodes and 30 20TB nodes (ipp037 is used for dvo) -# 1 x W1 + 1 x W2 + 1 x W3 + 12 more nodes... - +# These are ordered oddly to prevent excessive targetting shuffles. ipphosts METADATA camera STR GPC1 - XY01 STR ipp005 - XY02 STR ipp005 + XY01 STR ipp010 + XY02 STR ipp011 XY03 STR ipp006 - XY04 STR ipp006 + XY04 STR ipp054 XY05 STR ipp007 - XY06 STR ipp007 + XY06 STR ipp055 XY10 STR ipp008 - XY11 STR ipp008 + XY11 STR ipp056 XY12 STR ipp009 - XY13 STR ipp009 - XY14 STR ipp010 - XY15 STR ipp011 + XY13 STR ipp057 + XY14 STR ipp058 + XY15 STR ipp059 XY16 STR ipp012 - XY17 STR ipp012 + XY17 STR ipp060 XY20 STR ipp013 XY21 STR ipp014 - XY22 STR ipp014 + XY22 STR ipp061 XY23 STR ipp015 XY24 STR ipp016 - XY25 STR ipp016 - - XY26 STR ipp017 - XY27 STR ipp018 - XY30 STR ipp019 + XY25 STR ipp062 + + XY26 STR ipp064 + XY27 STR ipp065 + XY30 STR ipp066 XY31 STR ipp020 - XY32 STR ipp021 + XY32 STR ipp063 XY33 STR ipp021 XY34 STR ipp023 - XY35 STR ipp023 + XY35 STR ipp013 XY36 STR ipp024 - XY37 STR ipp024 + XY37 STR ipp019 XY40 STR ipp025 - XY41 STR ipp025 - - XY42 STR ipp026 - XY43 STR ipp026 + XY41 STR ipp018 + + XY42 STR ipp017 + XY43 STR ipp015 XY44 STR ipp027 XY45 STR ipp028 @@ -174,74 +190,74 @@ camera STR gpc1 - ota01 STR ipp005 - ota02 STR ipp005 + ota01 STR ipp010 + ota02 STR ipp011 ota03 STR ipp006 - ota04 STR ipp006 - + ota04 STR ipp054 + ota05 STR ipp007 - ota06 STR ipp007 + ota06 STR ipp055 ota10 STR ipp008 - ota11 STR ipp008 - + ota11 STR ipp056 + ota12 STR ipp009 - ota13 STR ipp009 - ota14 STR ipp010 - ota15 STR ipp011 - + ota13 STR ipp057 + ota14 STR ipp058 + ota15 STR ipp059 + ota16 STR ipp012 - ota17 STR ipp012 + ota17 STR ipp060 ota20 STR ipp013 ota21 STR ipp014 - - ota22 STR ipp014 + + ota22 STR ipp061 ota23 STR ipp015 ota24 STR ipp016 - ota25 STR ipp016 - - ota26 STR ipp017 - ota27 STR ipp018 - ota30 STR ipp019 + ota25 STR ipp062 + + ota26 STR ipp064 + ota27 STR ipp065 + ota30 STR ipp066 ota31 STR ipp020 - - ota32 STR ipp021 + + ota32 STR ipp063 ota33 STR ipp021 ota34 STR ipp023 - ota35 STR ipp023 - + ota35 STR ipp013 + ota36 STR ipp024 - ota37 STR ipp024 + ota37 STR ipp019 ota40 STR ipp025 - ota41 STR ipp025 - - ota42 STR ipp026 - ota43 STR ipp026 + ota41 STR ipp018 + + ota42 STR ipp017 + ota43 STR ipp015 ota44 STR ipp027 ota45 STR ipp028 - + ota46 STR ipp029 ota47 STR ipp030 ota50 STR ipp031 ota51 STR ipp032 - + ota52 STR ipp033 ota53 STR ipp034 ota54 STR ipp035 ota55 STR ipp036 - + ota56 STR ipp037 ota57 STR ipp038 ota60 STR ipp039 ota61 STR ipp040 - + ota62 STR ipp041 ota63 STR ipp042 ota64 STR ipp043 ota65 STR ipp044 - + ota66 STR ipp046 ota67 STR ipp047 ota71 STR ipp048 ota72 STR ipp049 - + ota73 STR ipp050 ota74 STR ipp051 @@ -285,74 +301,74 @@ count S32 60 - XY01 STR ipp005 - XY02 STR ipp005 + XY01 STR ipp010 + XY02 STR ipp011 XY03 STR ipp006 - XY04 STR ipp006 - + XY04 STR ipp054 + XY05 STR ipp007 - XY06 STR ipp007 + XY06 STR ipp055 XY10 STR ipp008 - XY11 STR ipp008 - + XY11 STR ipp056 + XY12 STR ipp009 - XY13 STR ipp009 - XY14 STR ipp010 - XY15 STR ipp011 - + XY13 STR ipp057 + XY14 STR ipp058 + XY15 STR ipp059 + XY16 STR ipp012 - XY17 STR ipp012 + XY17 STR ipp060 XY20 STR ipp013 XY21 STR ipp014 - - XY22 STR ipp014 + + XY22 STR ipp061 XY23 STR ipp015 XY24 STR ipp016 - XY25 STR ipp016 - - XY26 STR ipp017 - XY27 STR ipp018 - XY30 STR ipp019 + XY25 STR ipp062 + + XY26 STR ipp064 + XY27 STR ipp065 + XY30 STR ipp066 XY31 STR ipp020 - - XY32 STR ipp021 + + XY32 STR ipp063 XY33 STR ipp021 XY34 STR ipp023 - XY35 STR ipp023 - + XY35 STR ipp013 + XY36 STR ipp024 - XY37 STR ipp024 + XY37 STR ipp019 XY40 STR ipp025 - XY41 STR ipp025 - - XY42 STR ipp026 - XY43 STR ipp026 + XY41 STR ipp018 + + XY42 STR ipp017 + XY43 STR ipp015 XY44 STR ipp027 XY45 STR ipp028 - + XY46 STR ipp029 XY47 STR ipp030 XY50 STR ipp031 XY51 STR ipp032 - + XY52 STR ipp033 XY53 STR ipp034 XY54 STR ipp035 XY55 STR ipp036 - + XY56 STR ipp037 XY57 STR ipp038 XY60 STR ipp039 XY61 STR ipp040 - + XY62 STR ipp041 XY63 STR ipp042 XY64 STR ipp043 XY65 STR ipp044 - + XY66 STR ipp046 XY67 STR ipp047 XY71 STR ipp048 XY72 STR ipp049 - + XY73 STR ipp050 XY74 STR ipp051 @@ -363,80 +379,250 @@ ipphosts METADATA camera STR dist_skycell - count S32 76 - sky00 STR ipp005 + count S32 86 + sky00 STR ipp006 sky01 STR ipp006 sky02 STR ipp007 - sky03 STR ipp008 - sky04 STR ipp009 - sky05 STR ipp010 - sky06 STR ipp011 - sky07 STR ipp012 - sky08 STR ipp012 - sky09 STR ipp013 - sky10 STR ipp014 - sky11 STR ipp014 - sky12 STR ipp015 - sky13 STR ipp016 - sky14 STR ipp016 - sky15 STR ipp017 - sky16 STR ipp018 - sky17 STR ipp019 - sky18 STR ipp020 - sky19 STR ipp021 - sky20 STR ipp023 - sky21 STR ipp024 - sky22 STR ipp025 - sky23 STR ipp026 - sky24 STR ipp027 - sky25 STR ipp027 - sky26 STR ipp028 - sky27 STR ipp028 - sky28 STR ipp029 - sky29 STR ipp029 - sky30 STR ipp030 - sky31 STR ipp030 - sky32 STR ipp031 - sky33 STR ipp031 - sky34 STR ipp032 - sky35 STR ipp032 - sky36 STR ipp033 - sky37 STR ipp033 - sky38 STR ipp034 - sky39 STR ipp034 - sky40 STR ipp035 - sky41 STR ipp035 - sky42 STR ipp036 + sky03 STR ipp007 + sky04 STR ipp008 + sky05 STR ipp008 + sky06 STR ipp009 + sky07 STR ipp009 + sky08 STR ipp010 + sky09 STR ipp010 + sky10 STR ipp011 + sky11 STR ipp011 + sky12 STR ipp012 + sky13 STR ipp012 + sky14 STR ipp013 + sky15 STR ipp013 + sky16 STR ipp014 + sky17 STR ipp014 + sky18 STR ipp015 + sky19 STR ipp015 + sky20 STR ipp016 + sky21 STR ipp016 + sky22 STR ipp017 + sky23 STR ipp017 + sky24 STR ipp018 + sky25 STR ipp018 + sky26 STR ipp019 + sky27 STR ipp019 + sky28 STR ipp020 + sky29 STR ipp020 + sky30 STR ipp021 + sky31 STR ipp021 + sky32 STR ipp023 + sky33 STR ipp024 + sky34 STR ipp025 + sky35 STR ipp027 + sky36 STR ipp028 + sky37 STR ipp030 + sky38 STR ipp031 + sky39 STR ipp032 + sky40 STR ipp033 + sky41 STR ipp034 + sky42 STR ipp035 sky43 STR ipp036 sky44 STR ipp037 - sky45 STR ipp037 - sky46 STR ipp038 - sky47 STR ipp038 - sky48 STR ipp039 - sky49 STR ipp039 - sky50 STR ipp040 - sky51 STR ipp040 - sky52 STR ipp041 - sky53 STR ipp041 - sky54 STR ipp042 - sky55 STR ipp042 - sky56 STR ipp043 - sky57 STR ipp043 - sky58 STR ipp044 - sky59 STR ipp044 - sky60 STR ipp046 - sky61 STR ipp046 - sky62 STR ipp047 - sky63 STR ipp047 - sky64 STR ipp048 - sky65 STR ipp048 - sky66 STR ipp049 - sky67 STR ipp049 - sky68 STR ipp050 - sky69 STR ipp050 - sky70 STR ipp051 - sky71 STR ipp051 - sky72 STR ipp052 - sky73 STR ipp052 - sky74 STR ipp053 - sky75 STR ipp053 + sky45 STR ipp038 + sky46 STR ipp039 + sky47 STR ipp040 + sky48 STR ipp041 + sky49 STR ipp042 + sky50 STR ipp043 + sky51 STR ipp044 + sky52 STR ipp046 + sky53 STR ipp047 + sky54 STR ipp048 + sky55 STR ipp049 + sky56 STR ipp050 + sky57 STR ipp051 + sky58 STR ipp052 + sky59 STR ipp053 + sky60 STR ipp054 + sky61 STR ipp054 + sky62 STR ipp055 + sky63 STR ipp055 + sky64 STR ipp056 + sky65 STR ipp056 + sky66 STR ipp057 + sky67 STR ipp057 + sky68 STR ipp058 + sky69 STR ipp058 + sky70 STR ipp059 + sky71 STR ipp059 + sky72 STR ipp060 + sky73 STR ipp060 + sky74 STR ipp061 + sky75 STR ipp061 + sky76 STR ipp062 + sky77 STR ipp062 + sky78 STR ipp063 + sky79 STR ipp063 + sky80 STR ipp064 + sky81 STR ipp064 + sky82 STR ipp065 + sky83 STR ipp065 + sky84 STR ipp066 + sky85 STR ipp066 END + +ipphosts METADATA + camera STR STARE + + ota01 STR stare00 + ota02 STR stare01 + ota03 STR stare02 + ota04 STR stare04 + + ota05 STR stare00 + ota06 STR stare01 + ota10 STR stare02 + ota11 STR stare04 + + ota12 STR stare00 + ota13 STR stare01 + ota14 STR stare02 + ota15 STR stare04 + + ota16 STR stare00 + ota17 STR stare01 + ota20 STR stare02 + ota21 STR stare04 + + ota22 STR stare00 + ota23 STR stare01 + ota24 STR stare02 + ota25 STR stare04 + + ota26 STR stare00 + ota27 STR stare01 + ota30 STR stare02 + ota31 STR stare04 + + ota32 STR stare00 + ota33 STR stare01 + ota34 STR stare02 + ota35 STR stare04 + + ota36 STR stare00 + ota37 STR stare01 + ota40 STR stare02 + ota41 STR stare04 + + ota42 STR stare00 + ota43 STR stare01 + ota44 STR stare02 + ota45 STR stare04 + + ota46 STR stare00 + ota47 STR stare01 + ota50 STR stare02 + ota51 STR stare04 + + ota52 STR stare00 + ota53 STR stare01 + ota54 STR stare02 + ota55 STR stare04 + + ota56 STR stare00 + ota57 STR stare01 + ota60 STR stare02 + ota61 STR stare04 + + ota62 STR stare00 + ota63 STR stare01 + ota64 STR stare02 + ota65 STR stare04 + + ota66 STR stare00 + ota67 STR stare01 + ota71 STR stare02 + ota72 STR stare04 + + ota73 STR stare00 + ota74 STR stare01 + ota75 STR stare02 + ota76 STR stare04 +END + + +# Backup of old stare allocation setup. +# ipphosts METADATA +# camera STR STARE + +# ota01 STR ipp010 +# ota02 STR ipp010 +# ota03 STR ipp010 +# ota04 STR ipp010 + +# ota05 STR ipp011 +# ota06 STR ipp011 +# ota10 STR ipp011 +# ota11 STR ipp011 + +# ota12 STR ipp012 +# ota13 STR ipp012 +# ota14 STR ipp012 +# ota15 STR ipp012 + +# ota16 STR ipp013 +# ota17 STR ipp013 +# ota20 STR ipp013 +# ota21 STR ipp013 + +# ota22 STR ipp014 +# ota23 STR ipp014 +# ota24 STR ipp014 +# ota25 STR ipp014 + +# ota26 STR ipp015 +# ota27 STR ipp015 +# ota30 STR ipp015 +# ota31 STR ipp015 + +# ota32 STR ipp016 +# ota33 STR ipp016 +# ota34 STR ipp016 +# ota35 STR ipp016 + +# ota36 STR ipp017 +# ota37 STR ipp017 +# ota40 STR ipp017 +# ota41 STR ipp017 + +# ota42 STR ipp018 +# ota43 STR ipp018 +# ota44 STR ipp018 +# ota45 STR ipp018 + +# ota46 STR ipp019 +# ota47 STR ipp019 +# ota50 STR ipp019 +# ota51 STR ipp019 + +# ota52 STR ipp020 +# ota53 STR ipp020 +# ota54 STR ipp020 +# ota55 STR ipp020 + +# ota56 STR ipp021 +# ota57 STR ipp021 +# ota60 STR ipp021 +# ota61 STR ipp021 + +# ota62 STR ipp010 +# ota63 STR ipp011 +# ota64 STR ipp012 +# ota65 STR ipp013 + +# ota66 STR ipp014 +# ota67 STR ipp015 +# ota71 STR ipp016 +# ota72 STR ipp017 + +# ota73 STR ipp018 +# ota74 STR ipp019 +# ota75 STR ipp020 +# ota76 STR ipp021 +# END Index: branches/meh_branches/ppstack_test/ippTasks/lap.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/lap.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/lap.pro (revision 33415) @@ -10,4 +10,6 @@ $lap_RunPage = 0 $lap_DonePage = 0 + +$LAP_QUEUE = NULL book init lapNewRuns @@ -135,4 +137,8 @@ +macro lap.define.queue + $LAP_QUEUE = $1 +end + task lap.initial.load host local @@ -388,5 +394,5 @@ stderr $LOGDIR/lap.load.log - $run = laptool -pendingrun -state done + $run = laptool -pendingrun -state full if ($lapSeq:n != 0) @@ -463,4 +469,8 @@ option $LAP_ID $run = lap_science.pl --cleanup_mode --dbname $DBNAME --lap_id $LAP_ID + if ("$LAP_QUEUE" != "NULL") + $run = $run --queue_list $LAP_QUEUE + end + command $run Index: branches/meh_branches/ppstack_test/ippTasks/mergedvodb.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/mergedvodb.pro (revision 33415) +++ branches/meh_branches/ppstack_test/ippTasks/mergedvodb.pro (revision 33415) @@ -0,0 +1,226 @@ +## addstar.pro : globals and support macros : -*- sh -*- +## this file contains the tasks for running the addstar analysis stage +## these tasks use the book addPendingExp + +# test for required global variables +check.globals + + +#There is a book for each task, because I don't use labels. + +if (not($?havemergedvodbBooks)) + book create MERGEDVODB_MERGE + $havemergedvodbBooks = TRUE +end + +$MERGEDVODB_DB = 0 +$MERGEDVODB_MERGE_DB = 0 + +book init mergedvodbMergelist + +macro mergedvodb.merge.status + book listbook mergedvodbMergelist +end + +macro mergedvodb.merge.reset + book init mergedvodbMergelist +end + +macro mergedvodb.merge.on + task mergedvodb.merge.load + active true + end + task mergedvodb.merge.run + active true + end +end +macro mergedvodb.merge.off + task mergedvodb.merge.load + active false + end + task mergedvodb.merge.run + active false + end +end + +## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. +macro add.mergedvodb + if ($0 != 2) + echo "USAGE: add.mergedvodb (mergedvodb)" + break + end + #merge + book newpage MERGEDVODB_MERGE $1 + book setword MERGEDVODB_MERGE $1 MERGEDVODB $1 + book setword MERGEDVODB_MERGE $1 STATE PENDING +end + +macro del.mergedvodb + if ($0 != 2) + echo "USAGE: del.mergedvodb (mergedvodb)" + break + end + book delpage MERGEDVODB_MERGE $1 +end + +macro show.mergedvodb + if ($0 != 1) + echo "USAGE: show.mergedvodb" + break + end + echo "mergedvodb merge" + book listbook MERGEDVODB_MERGE + echo "n pages" + book npages MERGEDVODB_MERGE -var N + echo $N +end + + +task mergedvodb.merge.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/mergedvodb.merge.load.log + + task.exec + book npages MERGEDVODB_MERGE -var N + if ($N == 0) + echo "No labels for processing" + break + endif + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + if ("$mergedvodb" == "NULL") + # All labels have been done --- reset + for i 0 $N + book getpage MERGEDVODB_MERGE $i -var mergedvodb + book setword MERGEDVODB_MERGE $mergedvodb STATE NEW + end + book getpage MERGEDVODB_MERGE 0 -var mergedvodb -key STATE NEW + + # Select different database + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + book setword MERGEDVODB_MERGE $mergedvodb STATE DONE + + $run = mergetool -pendingmerge -limit 1 + $run = $run -mergedvodb $mergedvodb + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run -dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + #add_poll_args run + #echo $run + command $run + end + + # success + task.exit 0 + #echo "success" + # convert 'stdout' to book format + ipptool2book stdout mergedvodbMergelist -key merge_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook mergedvodbMergelist + end + # delete existing entries in the appropriate pantaskStates + process_cleanup mergedvodbMergelist + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task mergedvodb.merge.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 60000 + + #we only want one running at a time + + host local + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $RUNEXEC + book npages mergedvodbMergelist -var N + if ($N == 0) break + # look for new images in minidvodbWaitlist (pantaskState == INIT) + book getpage mergedvodbMergelist 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + book setword mergedvodbMergelist $pageName pantaskState RUN + book getword mergedvodbMergelist $pageName merge_id -var MERGE_ID + book getword mergedvodbMergelist $pageName mergedvodb -var MERGEDVODB + book getword mergedvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH + book getword mergedvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword mergedvodbMergelist $pageName state -var STATE + stdout $LOGDIR/mergedvodb.merge.run.log + stderr $LOGDIR/mergedvodb.merge.run.log + $run = mergedvodb_merge.pl --camera GPC1 + $run = $run --mergedvodb_path $MERGEDVODB_PATH + $run = $run --minidvodb_path $MINIDVODB_PATH + $run = $run --merge_id $MERGE_ID --mergedvodb $MERGEDVODB + echo $run + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$MERGEDVODB_DB + $run = $run --dbname $DB:$MERGEDVODB_DB + $MERGEDVODB_DB ++ + if ($MERGEDVODB_DB >= $DB:n) set MERGEDVODB_DB = 0 + end + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + # if we are unable to run the 'exec', use a long retry time + periods -exec 0.05 + command $run + end + # default exit status + task.exit default + process_exit mergedvodbMergelist $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + + echo "hostname: $JOB_HOSTNAME" + + # Set a fault code in the database + + process_exit mergedvodbMergelist $options:0 $EXIT_CRASH_ERR + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword mergedvodbMergelist $options:0 pantaskState TIMEOUT + end +end + Index: branches/meh_branches/ppstack_test/ippTasks/minidvodb.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/minidvodb.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/minidvodb.pro (revision 33415) @@ -11,5 +11,5 @@ if (not($?haveminidvodbBooks)) book create MINIDVODB - book create MINIDVODB_MERGE + book create MINIDVODB_PREMERGE book create MINIDVODB_CREATE book create MINIDVODB_ACTIVE @@ -20,10 +20,10 @@ $MINIDVODB_WAIT_DB = 0 -$MINIDVODB_MERGE_DB = 0 +$MINIDVODB_PREMERGE_DB = 0 $MINIDVODB_CREATE_DB = 0 $MINIDVODB_ACTIVE_DB = 0 book init minidvodbWaitlist -book init minidvodbMergelist +book init minidvodbPreMergelist book init minidvodbCreatelist book init minidvodbActivelist @@ -54,10 +54,10 @@ end -macro minidvodb.merge.status - book listbook minidvodbMergelist -end - -macro minidvodb.merge.reset - book init minidvodbMergelist +macro minidvodb.premerge.status + book listbook minidvodbPreMergelist +end + +macro minidvodb.premerge.reset + book init minidvodbPreMergelist end @@ -112,17 +112,17 @@ ##these merge the dbs -macro minidvodb.merge.on - task minidvodb.merge.load +macro minidvodb.premerge.on + task minidvodb.premerge.load active true end - task minidvodb.merge.run + task minidvodb.premerge.run active true end end -macro minidvodb.merge.off - task minidvodb.merge.load +macro minidvodb.premerge.off + task minidvodb.premerge.load active false end - task minidvodb.merge.run + task minidvodb.premerge.run active false end @@ -142,8 +142,8 @@ book setword MINIDVODB $1 STATE PENDING #merge - book newpage MINIDVODB_MERGE $1 - book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 - book setword MINIDVODB_MERGE $1 DVODB $3 - book setword MINIDVODB_MERGE $1 STATE PENDING + book newpage MINIDVODB_PREMERGE $1 + book setword MINIDVODB_PREMERGE $1 MINIDVODB_GROUP $1 + book setword MINIDVODB_PREMERGE $1 DVODB $3 + book setword MINIDVODB_PREMERGE $1 STATE PENDING #active book newpage MINIDVODB_ACTIVE $1 @@ -168,5 +168,5 @@ end book delpage MINIDVODB $1 - book delpage MINIDVODB_MERGE $1 + book delpage MINIDVODB_PREMERGE $1 book delpage MINIDVODB_CREATE $1 book delpage MINIDVODB_ACTIVE $1 @@ -180,6 +180,6 @@ echo "minidvodb wait" book listbook MINIDVODB - echo "minidvodb merge" - book listbook MINIDVODB_MERGE + echo "minidvodb premerge" + book listbook MINIDVODB_PREMERGE echo "minidvodb create" book listbook MINIDVODB_CREATE @@ -339,5 +339,5 @@ -task minidvodb.merge.load +task minidvodb.premerge.load host local @@ -348,8 +348,8 @@ stdout NULL - stderr $LOGDIR/minidvodb.merge.load.log + stderr $LOGDIR/minidvodb.premerge.load.log task.exec - book npages MINIDVODB_MERGE -var N + book npages MINIDVODB_PREMERGE -var N if ($N == 0) # echo "No labels for processing" @@ -357,13 +357,13 @@ endif - book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW if ("$minidvodb_group" == "NULL") # All labels have been done --- reset # echo "Resetting labels" for i 0 $N - book getpage MINIDVODB_MERGE $i -var minidvodb_group - book setword MINIDVODB_MERGE $minidvodb_group STATE NEW + book getpage MINIDVODB_PREMERGE $i -var minidvodb_group + book setword MINIDVODB_PREMERGE $minidvodb_group STATE NEW end - book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW + book getpage MINIDVODB_PREMERGE 0 -var minidvodb_group -key STATE NEW # Select different database @@ -372,5 +372,5 @@ end #finds the minidvodbs in a state of 'to_be_merged' - book setword MINIDVODB_MERGE $minidvodb_group STATE DONE + book setword MINIDVODB_PREMERGE $minidvodb_group STATE DONE $run = addtool -listminidvodbrun -state to_be_merged -limit 1 $run = $run -minidvodb_group $minidvodb_group @@ -392,10 +392,10 @@ task.exit 0 # convert 'stdout' to book format - ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT + ipptool2book stdout minidvodbPreMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT if ($VERBOSE > 2) - book listbook minidvodbMergelist + book listbook minidvodbPreMergelist end # delete existing entries in the appropriate pantaskStates - process_cleanup minidvodbMergelist + process_cleanup minidvodbPreMergelist end @@ -418,5 +418,5 @@ -task minidvodb.merge.run +task minidvodb.premerge.run periods -poll $RUNPOLL periods -exec $RUNEXEC @@ -432,21 +432,20 @@ periods -exec $RUNEXEC - book npages minidvodbMergelist -var N + book npages minidvodbPreMergelist -var N if ($N == 0) break # look for new images in minidvodbWaitlist (pantaskState == INIT) - book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT + book getpage minidvodbPreMergelist 0 -var pageName -key pantaskState INIT if ("$pageName" == "NULL") break - book setword minidvodbMergelist $pageName pantaskState RUN - book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID - book getword minidvodbMergelist $pageName minidvodb_group -var MINIDVODB_GROUP - book getword minidvodbMergelist $pageName mergedvodb_path -var MERGEDVODB_PATH - book getword minidvodbMergelist $pageName minidvodb_path -var MINIDVODB_PATH - book getword minidvodbMergelist $pageName state -var STATE - stdout $LOGDIR/minidvodb.merge.run.log - stderr $LOGDIR/minidvodb.merge.run.log - - $run = minidvodb_merge.pl --camera GPC1 --mergedvodb $MERGEDVODB_PATH --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID + book setword minidvodbPreMergelist $pageName pantaskState RUN + book getword minidvodbPreMergelist $pageName minidvodb_id -var MINIDVODB_ID + book getword minidvodbPreMergelist $pageName minidvodb_group -var MINIDVODB_GROUP + book getword minidvodbPreMergelist $pageName minidvodb_path -var MINIDVODB_PATH + book getword minidvodbPreMergelist $pageName state -var STATE + stdout $LOGDIR/minidvodb.premerge.run.log + stderr $LOGDIR/minidvodb.premerge.run.log + + $run = minidvodb_premerge.pl --camera GPC1 --minidvodb $MINIDVODB_PATH --minidvodb_group $MINIDVODB_GROUP --minidvodb_id $MINIDVODB_ID if ($DB:n == 0) @@ -472,5 +471,5 @@ # default exit status task.exit default - process_exit minidvodbMergelist $options:0 $JOB_STATUS + process_exit minidvodbPreMergelist $options:0 $JOB_STATUS end @@ -482,5 +481,5 @@ # Set a fault code in the database - process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR + process_exit minidvodbPreMergelist $options:0 $EXIT_CRASH_ERR end @@ -488,5 +487,5 @@ task.exit timeout showcommand timeout - book setword minidvodbMergelist $options:0 pantaskState TIMEOUT + book setword minidvodbPreMergelist $options:0 pantaskState TIMEOUT end end Index: branches/meh_branches/ppstack_test/ippTasks/nightly_science.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/nightly_science.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/nightly_science.pro (revision 33415) @@ -178,5 +178,5 @@ periods -poll $LOADPOLL periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 300 trange 0:00:00 1:00:00 -nmax 1 npending 1 @@ -260,5 +260,5 @@ periods -poll 3600 periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 300 trange 22:00:00 23:59:59 trange 00:00:00 10:00:00 Index: branches/meh_branches/ppstack_test/ippTasks/nightly_stacks.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/nightly_stacks.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/nightly_stacks.pro (revision 33415) @@ -290,7 +290,7 @@ task ns.dqstats.load host local - periods -poll 3600 + periods -poll 10 periods -exec $LOADEXEC - periods -timeout 30 + periods -timeout 600 trange 22:00:00 23:59:59 trange 00:00:00 10:00:00 Index: branches/meh_branches/ppstack_test/ippTasks/pstamp.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/pstamp.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/pstamp.pro (revision 33415) @@ -626,5 +626,5 @@ periods -poll $LOADPOLL # XXX: create a macro for this time - periods -exec 10 + periods -exec 15 # periods -exec $LOADEXEC periods -timeout 300 @@ -676,9 +676,9 @@ task pstamp.dependent.run periods -poll $RUNPOLL - periods -exec $RUNEXEC + periods -exec 0.5 periods -timeout 300 task.exec - periods -exec $RUNEXEC + # periods -exec $RUNEXEC book npages pstampDependent -var N @@ -726,5 +726,5 @@ echo command $run end - periods -exec 0.05 +# periods -exec 0.05 command $run end Index: branches/meh_branches/ppstack_test/ippTasks/rcserver.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/rcserver.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/rcserver.pro (revision 33415) @@ -108,4 +108,5 @@ periods -exec $RUNEXEC periods -timeout 60 + npending 5 task.exec Index: branches/meh_branches/ppstack_test/ippTasks/skycal.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/skycal.pro (revision 33415) +++ branches/meh_branches/ppstack_test/ippTasks/skycal.pro (revision 33415) @@ -0,0 +1,246 @@ +## skycal.pro : tasks for static sky calibration analysis : -*- sh -*- + +## This file contains panTasks definitions for performing the +## skycal analysis. After a skycalRun entry (with +## associated skycal_id) is defined, psastro is performed (via script +## skycalibration.pl) (tasks in skycalRun). + +# test for required global variables +check.globals + +### Initialise the books containing the tasks to do +book init skycalRun + +### Database lists +$skycal_DB = 0 +$skycal_revert_DB = 0 + +### Check status of skycal tasks +macro skycal.status + book listbook skycalRun +end + +### Reset skycal tasks +macro skycal.reset + book init skycalRun +end + +### Turn skycal tasks on +macro skycal.on + task skycal.load + active true + end + task skycal.run + active true + end +end + +### Turn skycal tasks off +macro skycal.off + task skycal.load + active false + end + task skycal.run + active false + end +end + +macro skycal.revert.on + task skycal.revert + active true + end +end + +macro skycal.revert.off + task skycal.revert + active false + end +end + +### Load tasks for skycal +### Tasks are loaded into skycalRun. +task skycal.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGDIR/skycal.log + + task.exec + if ($LABEL:n == 0) break + $run = staticskytool -pendingskycalrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$skycal_DB + $run = $run -dbname $DB:$skycal_DB -limit 40 + $skycal_DB ++ + if ($skycal_DB >= $DB:n) set skycal_DB = 0 + end + add_poll_args run + add_poll_labels run + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout skycalRun -key skycal_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook skycalRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup skycalRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +### Run tasks for the skycal analysis +### Tasks are taken from skycalRun. +task skycal.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 10800 + + task.exec + book npages skycalRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new entries in skycalRun (pantaskState == INIT) + book getpage skycalRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword skycalRun $pageName pantaskState RUN + book getword skycalRun $pageName skycal_id -var SKYCAL_ID + book getword skycalRun $pageName stack_id -var STACK_ID + book getword skycalRun $pageName tess_id -var TESS_DIR + book getword skycalRun $pageName skycell_id -var SKYCELL_ID + book getword skycalRun $pageName camera -var CAMERA + book getword skycalRun $pageName filter -var FILTER + book getword skycalRun $pageName workdir -var WORKDIR_TEMPLATE + book getword skycalRun $pageName path_base -var PATH_BASE + book getword skycalRun $pageName reduction -var REDUCTION + book getword skycalRun $pageName singlefilter -var SINGLEFILTER + book getword skycalRun $pageName dbname -var DBNAME + book getword skycalRun $pageName state -var RUN_STATE + + # set the host and workdir based on the skycell hash + set.host.for.skycell $SKYCELL_ID + set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + basename $TESS_DIR -var TESS_ID + sprintf outroot "%s/%s/%s/%s.%s.stk.%s.skycal.%s" $WORKDIR $TESS_ID $SKYCELL_ID $TESS_ID $SKYCELL_ID $STACK_ID $SKYCAL_ID + + stdout $LOGDIR/skycal.log + stderr $LOGDIR/skycal.log + + $run = skycalibration.pl --skycal_id $SKYCAL_ID --outroot $outroot --redirect-output --camera $CAMERA --path_base $PATH_BASE --stack_id $STACK_ID --filter $FILTER + if ("$REDUCTION" != "NULL") + $run = $run --reduction $REDUCTION + end + if ("$SINGLEFILTER" != "0") + $run = $run --singlefilter + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit skycalRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword skycalRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword skycalRun $options:0 pantaskState TIMEOUT + end +end + +task skycal.revert + host local + + periods -poll 60.0 + periods -exec 1800.0 + periods -timeout 120.0 + npending 1 + active false + + stdout NULL + stderr $LOGDIR/revert.log + + task.exec + if ($LABEL:n == 0) break + # Only revert failures with fault=2 (SYS_ERROR), which tend to be + # temporary filesystem problems. Every other fault type is + # interesting and should be kept for debugging (and so it does not + # continue to occur). + $run = staticskytool -revertskycal -fault 2 + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$skycal_revert_DB + $run = $run -dbname $DB:$skycal_revert_DB + $skycal_revert_DB ++ + if ($skycal_revert_DB >= $DB:n) set skycal_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + Index: branches/meh_branches/ppstack_test/ippTasks/summit.copy.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/summit.copy.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/summit.copy.pro (revision 33415) @@ -29,4 +29,8 @@ # list of pzDownloadExps that have completed downloading book init pzPendingAdvance + +macro copy.status + book listbook pzPendingImfile +end macro copy.on @@ -442,5 +446,18 @@ # this function uses workdir_template, default_host, volume_template, volume_default, # it sets workdir and volume - set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + # Look to see if we have a stare type observation, and redirect it to an appropriate host + substr $EXP_NAME 10 1 EXPTYPE_KEY + if ("$EXPTYPE_KEY" == "a") + if ($NEBULOUS == 1) + $workdir_template = neb://@HOST@.1 + end + set.workdir.by.camera STARE $CLASS_ID $workdir_template $default_host workdir_base + else + if ($NEBULOUS == 1) + $workdir_template = neb://@HOST@.0 + end + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + end # figure out filename Index: branches/meh_branches/ppstack_test/ippTasks/survey.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/survey.pro (revision 32161) +++ branches/meh_branches/ppstack_test/ippTasks/survey.pro (revision 33415) @@ -3,22 +3,4 @@ # test for required global variables check.globals - -# there is a bug in 'getbook', use a different method below -# book getbook SURVEY_DIFF -var isBook -# if ("$isBook" != "SURVEY_DIFF") -# book create SURVEY_DIFF -# end -# book getbook SURVEY_MAGIC -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_MAGIC -# end -# book getbook SURVEY_DESTREAK -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_DESTREAK -# end -# book getbook SURVEY_DIST -var isBook -# if ("$isBook" == "NULL") -# book create SURVEY_DIST -# end if (not($?haveSurveyBooks)) @@ -30,8 +12,9 @@ book create SURVEY_DIST book create SURVEY_ADDSTAR + book create SURVEY_MERGEDVODB book create SURVEY_CHIP_BG book create SURVEY_WARP_BG book create SURVEY_PUBLISH - book create SURVEY_STATICSKYSINGLE + book create SURVEY_STATICSKYSINGLE $haveSurveyBooks = TRUE end @@ -44,9 +27,9 @@ $SURVEY_DIST_DB = 0 $SURVEY_ADDSTAR_DB = 0 +$SURVEY_MERGEDVODB_DB = 0 $SURVEY_CHIP_BG_DB = 0 $SURVEY_WARP_BG_DB = 0 $SURVEY_PUBLISH_DB = 0 $SURVEY_STATICSKYSINGLE_DB = 0 - $SURVEY_EXEC = 120 $SURVEY_POLL = 10 @@ -69,4 +52,7 @@ active true end + task survey.mergedvodb + active true + end task survey.destreak active true @@ -105,4 +91,7 @@ active false end + task survey.mergedvodb + active false + end task survey.destreak active false @@ -213,10 +202,15 @@ # user functions to manipulate magic labels macro survey.add.magic - if ($0 != 3) - echo "USAGE: survey.add.magic (label) (workdir base)" + if (($0 != 3)&&($0 != 4)) + echo "USAGE: survey.add.magic (label) (workdir base) (multidiff)" break end book newpage SURVEY_MAGIC $1 book setword SURVEY_MAGIC $1 WORKDIR $2 + if ($0 == 3) + book setword SURVEY_MAGIC $1 MULTIDIFF FALSE + else + book setword SURVEY_MAGIC $1 MULTIDIFF $3 + end book setword SURVEY_MAGIC $1 STATE PENDING end @@ -267,4 +261,31 @@ end +macro survey.add.mergedvodb + if ($0 != 4) + echo "USAGE: survey.add.mergedvodb (tag) (mergedvodb) (minidvodb_group)" + break + end + book newpage SURVEY_MERGEDVODB $1 + book setword SURVEY_MERGEDVODB $1 MERGEDVODB $2 + book setword SURVEY_MERGEDVODB $1 MINIDVODB_GROUP $3 +end + +macro survey.del.mergedvodb + if ($0 != 2) + echo "USAGE: survey.del.mergedvodb (tag)" + break + end + book delpage SURVEY_MERGEDVODB $1 +end + +macro survey.show.mergedvodb + if ($0 != 1) + echo "USAGE: survey.show.mergedvodb" + break + end + book listbook SURVEY_MERGEDVODB +end + + # user functions to manipulate destreak labels macro survey.add.destreak @@ -297,10 +318,11 @@ # user functions to manipulate dist labels macro survey.add.dist - if ($0 != 3) - echo "USAGE: survey.add.dist (label) (workdir)" + if ($0 != 4) + echo "USAGE: survey.add.dist (label) (workdir) (muggle)" break end book newpage SURVEY_DIST $1 book setword SURVEY_DIST $1 WORKDIR $2 + book setword SURVEY_DIST $1 MUGGLE $3 book setword SURVEY_DIST $1 STATE PENDING end @@ -324,10 +346,11 @@ # user functions to manipulate chip_bg labels macro survey.add.chip.bg - if ($0 != 3) - echo "USAGE: survey.add.chip.bg (label) (dist_group)" + if ($0 != 4) + echo "USAGE: survey.add.chip.bg (label) (dist_group) (muggle)" break end book newpage SURVEY_CHIP_BG $1 book setword SURVEY_CHIP_BG $1 DIST_GROUP $2 + book setword SURVEY_CHIP_BG $1 MUGGLE $3 book setword SURVEY_CHIP_BG $1 STATE PENDING end @@ -351,10 +374,11 @@ # user functions to manipulate warp_bg labels macro survey.add.warp.bg - if ($0 != 3) - echo "USAGE: survey.add.warp.bg (label) (dist_group)" + if ($0 != 4) + echo "USAGE: survey.add.warp.bg (label) (dist_group) (muggle)" break end book newpage SURVEY_WARP_BG $1 book setword SURVEY_WARP_BG $1 DIST_GROUP $2 + book setword SURVEY_WARP_BG $1 MUGGLE $3 book setword SURVEY_WARP_BG $1 STATE PENDING end @@ -610,5 +634,8 @@ periods -exec $SURVEY_EXEC periods -timeout $SURVEY_TIMEOUT - trange 13:00:00 14:00:00 +# 13:00 was being done in early morning rather than afternoon HST (so UT value), want both times to have SSdiff ASAP? +# changing earlier for QUB to fetch @10pm for nightly processing there, should be enough time to finish MD stacks +# trange 13:00:00 14:00:00 + trange 20:00:00 21:00:00 npending 1 @@ -725,6 +752,11 @@ book setword SURVEY_MAGIC $label STATE DONE book getword SURVEY_MAGIC $label WORKDIR -var workdir - + book getword SURVEY_MAGIC $label MULTIDIFF -var multidiff + $run = magictool -rerun -definebyquery -label $label -diff_label $label -workdir $workdir/$label + + if (("$multidiff" == "TRUE")||("$multidiff" == "MULTIDIFF")) + $run = $run -multidiff + end if ($DB:n == 0) @@ -735,5 +767,5 @@ end - # echo $run + echo $run command $run end @@ -814,11 +846,12 @@ if ("$stage" == "cam") #only queue destreaked cams. stacks and staticsky don't need this - if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") + # if ("$DB:$SURVEY_ADDSTAR_DB" == "isp") #this is the only way I can think of how to handle this (but it is messy). If it is a database that we KNOW doesn't use magicked (ie, isp), then do not queue destreaked + # now with no magic, all are uncensored $run = $run -uncensored - else +# else #if not isp, then run destreaked version only - $run = $run -destreaked - end +# $run = $run -destreaked +# end end if ("$stage" == "staticsky") @@ -826,4 +859,9 @@ $run = $run -uncensored end + if ("$stage" == "staticsky_multi") + #only queue uncensored staticsky (multi filter) + $run = $run -uncensored + end + if ("$stage" == "stack") #only queue uncensored stacks @@ -858,4 +896,82 @@ end end + + +task survey.mergedvodb + host local + + periods -poll $SURVEY_POLL + periods -exec $SURVEY_EXEC + periods -timeout $SURVEY_TIMEOUT + npending 1 + + stdout $LOGDIR/survey.mergedvodb.log + stderr $LOGDIR/survey.mergedvodb.log + + task.exec + book npages SURVEY_MERGEDVODB -var N + if ($N == 0) +# echo "No labels for processing" + break + endif + + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + if ("$tag" == "NULL") + # All labels have been done --- reset + # echo "Resetting labels" + for i 0 $N + book getpage SURVEY_MERGEDVODB $i -var tag + book setword SURVEY_MERGEDVODB $tag STATE NEW + end + book getpage SURVEY_MERGEDVODB 0 -var tag -key STATE NEW + + # Select different database + $SURVEY_MERGEDVODB_DB ++ + if ($SURVEY_MERGEDVODB_DB >= $DB:n) set SURVEY_MERGEDVODB_DB = 0 + end + book setword SURVEY_MERGEDVODB $tag STATE DONE + book getword SURVEY_MERGEDVODB $tag MERGEDVODB -var mergedvodb + book getword SURVEY_MERGEDVODB $tag MINIDVODB_GROUP -var minidvodb_group + + # $run = addtool -definebyquery -destreaked -label $label -set_dvodb $dbodb + #$run = mergetool + $run = mergedvodb_queue.pl --outroot $mergedvodb --mergedvodb $mergedvodb --minidvodb_group $minidvodb_group + if ($DB:n == 0) + option DEFAULT + else + $run = $run --dbname $DB:$SURVEY_MERGEDVODB_DB + option $DB:$SURVEY_MERGEDVODB_DB + end + #this run doesn't work because there's no workdir + #$run = $run -definebyquery -mergedvodb $mergedvodb -minidvodb_group $minidvodb_group + + + # echo $run + + echo $run + command $run + end + + # success + + task.exit 0 +# echo "Success" + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + @@ -974,8 +1090,13 @@ $workdir = $workdir_base/$label/$year/$month/$day + book getword SURVEY_DIST $label MUGGLE -var muggle + book setword SURVEY_DIST $label STATE DONE # note workdir is set by the script based on site.config $run = dist_defineruns.pl --label $label --workdir $workdir + if ($muggle != 0) + $run = $run --no_magic + end if ($DB:n == 0) @@ -1048,6 +1169,10 @@ book setword SURVEY_CHIP_BG $label STATE DONE book getword SURVEY_CHIP_BG $label DIST_GROUP -var dist_group + book getword SURVEY_CHIP_BG $label MUGGLE -var muggle $run = bgtool -definechip -label $label -set_dist_group $dist_group + if ($muggle == 0) + $run = $run -destreaked + end if ($DB:n == 0) @@ -1058,5 +1183,7 @@ end - # echo $run + if ($VERBOSE > 1) + echo $run + end command $run end @@ -1118,6 +1245,11 @@ book setword SURVEY_WARP_BG $label STATE DONE book getword SURVEY_WARP_BG $label DIST_GROUP -var dist_group + book getword SURVEY_WARP_BG $label MUGGLE -var muggle $run = bgtool -definewarp -warp_label $label -set_dist_group $dist_group + if ($muggle == 0) + $run = $run -destreaked + end + if ($DB:n == 0) @@ -1128,5 +1260,7 @@ end - # echo $run + if ($VERBOSE > 1) + echo $run + end command $run end @@ -1248,5 +1382,5 @@ book npages SURVEY_STATICSKYSINGLE -var N if ($N == 0) - echo "No STATICSKYSINGLE labels for processing" + # echo "No STATICSKYSINGLE labels for processing" break endif Index: branches/meh_branches/ppstack_test/ippTasks/vp.pro =================================================================== --- branches/meh_branches/ppstack_test/ippTasks/vp.pro (revision 33415) +++ branches/meh_branches/ppstack_test/ippTasks/vp.pro (revision 33415) @@ -0,0 +1,249 @@ +## vp.pro : tasks for video photometry : -*- sh -*- + +# test for required global variables +check.globals + +$LOGSUBDIR = $LOGDIR/vp +mkdir $LOGSUBDIR + +### Initialise the books containing the tasks to do +book init vpPendingRun + +### Database lists +$vp_DB = 0 +$vp_revert_DB = 0 + +$VP_POLL_LIMIT = 20 + +### Check status of vping tasks +macro vp.status + book listbook vpPendingRun +end + +### Reset vping tasks +macro vp.reset + book init vpPendingRun +end + +### Turn vp tasks on +macro vp.on + task vp.load + active true + end + task vp.run + active true + end +end + +### Turn vp tasks off +macro vp.off + task vp.load + active false + end + task vp.run + active false + end +end + +macro vp.revert.on + task vp.revert + active true + end +end + +macro vp.revert.off + task vp.revert + active false + end +end + +macro set.vp.poll + $VP_POLL_LIMIT = $1 +end +macro get.vp.poll + echo $VP_POLL_LIMIT +end + + +### Load tasks for doing video photometry +### Tasks are loaded into vpPendingRun. +task vp.load + host local + + periods -poll $LOADPOLL + periods -exec $LOADEXEC + periods -timeout 30 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/vp.load.log + + task.exec + if ($LABEL:n == 0) break + $run = vptool -pendingrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$vp_DB + $run = $run -dbname $DB:$vp_DB + $vp_DB ++ + if ($vp_DB >= $DB:n) set vp_DB = 0 + end + add_poll_args run + add_poll_labels run + $run = $run -limit $VP_POLL_LIMIT + command $run + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout vpPendingRun -key vp_id -uniq -setword dbname $options:0 -setword pantaskState INIT + if ($VERBOSE > 2) + book listbook vpPendingRun + end + + # delete existing entries in the appropriate pantaskStates + process_cleanup vpPendingRun + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + + + + +### Run video photometery tasks +### Tasks are taken from vpPendingRun. +task vp.run + periods -poll $RUNPOLL + periods -exec $RUNEXEC + periods -timeout 300 + + task.exec + book npages vpPendingRun -var N + if ($N == 0) break + if ($NETWORK == 0) break + if ($BURNTOOLING == 1) break + + # look for new runs in vpPendingRun (pantaskState == INIT) + book getpage vpPendingRun 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword vpPendingRun $pageName pantaskState RUN + book getword vpPendingRun $pageName vp_id -var VP_ID + book getword vpPendingRun $pageName camera -var CAMERA + book getword vpPendingRun $pageName exp_tag -var EXP_TAG + book getword vpPendingRun $pageName workdir -var WORKDIR + book getword vpPendingRun $pageName dest_id -var DEST_ID + book getword vpPendingRun $pageName dest_name -var DEST_NAME + book getword vpPendingRun $pageName ds_dbname -var DS_DBNAME + book getword vpPendingRun $pageName ds_dbhost -var DS_DBHOST + book getword vpPendingRun $pageName dbname -var DBNAME + + # set the host and workdir based on the skycell hash +# set.host.for.skycell $SKYCELL_ID +# set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR + + sprintf outroot "%s/%s.vp.%d" $WORKDIR $EXP_TAG $VP_ID + + stdout $LOGSUBDIR/vp.log + stderr $LOGSUBDIR/vp.log + + host anyhost + + $run = videophot_process.pl --vp_id $VP_ID --outroot $outroot --redirect-output --camera $CAMERA + if ($DEST_ID) + $run = $run --dest_id $DEST_ID --product $DEST_NAME --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST + end + add_standard_args run + + # save the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit vpPendingRun $options:0 $JOB_STATUS + end + + # locked list + task.exit crash + showcommand crash + echo "hostname: $JOB_HOSTNAME" + book setword vpPendingRun $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword vpPendingRun $options:0 pantaskState TIMEOUT + end +end + + +task vp.revert + host local + + periods -poll 5 + periods -exec 600 + periods -exec 30 + periods -timeout 120 + npending 1 + + stdout NULL + stderr $LOGSUBDIR/vp.revert.log + + task.exec + if ($LABEL:n == 0) break + $run = vptool -revertrun + if ($DB:n == 0) + option DEFAULT + else + # save the DB name for the exit tasks + option $DB:$vp_revert_DB + $run = $run -dbname $DB:$vp_revert_DB + $vp_revert_DB ++ + if ($vp_revert_DB >= $DB:n) set vp_revert_DB = 0 + end + add_poll_labels run + command $run + end + + # success + task.exit 0 + end + + # locked list + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: branches/meh_branches/ppstack_test/ippToPsps/config/IN/tables.vot =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/config/IN/tables.vot (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/config/IN/tables.vot (revision 33415) @@ -22,5 +22,4 @@ - @@ -98996,3077 +98995,3077 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/config/settings.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/config/settings.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/config/settings.xml (revision 33415) @@ -0,0 +1,42 @@ + + + + + + + + /data/ipp005.0/rhenders + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/batch.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/batch.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/batch.py (revision 33415) @@ -21,5 +21,5 @@ ''' -Base class of all batch types. +Abstract base class of all batch types. ''' class Batch(object): @@ -27,25 +27,22 @@ ''' Constructor - - >>> batch = Batch(1,2,3,4,5,6,7) - >>> print batch.pspsVoTableFilePath - "../config/2/tables.vot" ''' def __init__(self, logger, - configPath, - doc, + config, gpc1Db, ippToPspsDb, + scratchDb, id, + batchID, batchType, - fits): + fits, + useFullTables): self.everythingOK = False self.readHeader = False - self.configPath = configPath - self.doc = doc + self.config = config self.fits = fits - if self.fits: self.header = self.fits.getPrimaryHeader() + self.useFullTables = useFullTables # set up logging @@ -56,55 +53,50 @@ # set up class variables self.id = id + self.batchID = batchID self.gpc1Db = gpc1Db self.ippToPspsDb = ippToPspsDb + self.scratchDb = scratchDb self.batchType = batchType; self.pspsVoTableFilePath = "../config/" + batchType + "/tables.vot" - # get info from config - self.survey = self.doc.find("options/survey").text - self.pspsSurvey = self.doc.find("options/pspsSurvey").text - dvoGpc1Label = self.doc.find("dvo/gpc1Label").text - self.dvoLocation = self.doc.find("dvo/location").text - self.useFullTables = int(self.doc.find("dvo/useFullTables").text) - self.scratchDb = ScratchDb(logger, self.doc, self.useFullTables) - - if not self.scratchDb.everythingOK: return + # check FITS file is ok + if self.fits: + self.header = self.fits.getPrimaryHeader() + + # now check that the fits header is readable + if not self.header: + logger.errorPair("Could not read FITS for id", "%d" % id) + return + + # if no fits file, and this is not an IN batch, then get out of here + else: + if batchType != "IN": + logger.errorPair("Could not read FITS for id", "%d" % id) + return + + self.scratchDb.setUseFullTables(self.useFullTables) # TODO self.tablesToExport = [] - if self.survey != "": - self.surveyID = self.scratchDb.getSurveyID(self.survey) + if self.config.survey != "": + self.surveyID = self.scratchDb.getSurveyID(self.config.survey) else: self.surveyID = -1; - - # get some options from the config - self.testMode = int(self.doc.find("options/testMode").text) - self.dataRelease = int(self.doc.find("metadata/dataRelease").text) - # create datastore object - self.datastore = Datastore(self.logger, self.doc) - - # create a new batch - self.batchID = self.ippToPspsDb.createNewBatch( - self.batchType, - self.id, - self.survey, - dvoGpc1Label, - self.datastore.product) + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) # get local storage location from config self.batchName = Batch.getNameFromID(self.batchID) - basePath = self.doc.find("localOutPath").text self.subDir = Batch.getSubDir( - basePath, + self.config.basePath, self.batchType, - dvoGpc1Label) + self.config.dvoLabel) self.localOutPath = Batch.getOutputPath( - basePath, + self.config.basePath, self.batchType, - dvoGpc1Label, + self.config.dvoLabel, self.batchID) @@ -121,8 +113,5 @@ self.logger.infoTitle("New " + self.batchType + " batch") self.logger.infoPair("Batch name", self.batchName) - self.logger.infoPair("Survey", self.survey) self.logger.infoPair("Survey ID", "%d" % self.surveyID) - self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey) - self.logger.infoPair("DVO location", self.dvoLocation) self.logger.infoBool("Use full DVO tables?", self.useFullTables) @@ -157,4 +146,54 @@ ''' + Static method to get name of tar file + ''' + @staticmethod + def getTarFile(id): + return Batch.getNameFromID(id) + ".tar" + + ''' + Static method to get path of tar file + ''' + @staticmethod + def getTarPath(basePath, batchType, dvoLabel, id): + return Batch.getSubDir(basePath, batchType, dvoLabel) + "/" + Batch.getTarFile(id) + + ''' + Static method to get name of tarball file + ''' + @staticmethod + def getTarballFile(id): + return Batch.getTarFile(id) + ".gz" + + ''' + Static method to get path of tarball file + ''' + @staticmethod + def getTarballPath(basePath, batchType, dvoLabel, id): + return Batch.getSubDir(basePath, batchType, dvoLabel) + "/" + Batch.getTarballFile(id) + + ''' + Static method to delete this batch from local disk + ''' + @staticmethod + def deleteFromDisk(logger, basePath, batchType, dvoLabel, id): + + tarballPath = Batch.getTarballPath(basePath, batchType, dvoLabel, id) + dirPath = Batch.getOutputPath(basePath, batchType, dvoLabel, id) + + if not os.path.exists(tarballPath): + logger.errorPair("Could not find", tarballPath) + return 1 + + try: + os.remove(tarballPath) + logger.infoPair("Deleted", tarballPath) + except: + logger.errorPair("Could not delete", tarballPath) + return 0 + + return 1 + + ''' Destructor ''' @@ -173,5 +212,5 @@ if value != "NULL": return value else: - if not self.testMode: return "NULL" + if not self.config.test: return "NULL" header[key] = default self.logger.infoPair("Hardcoding " + key + " to", header[key]) @@ -194,5 +233,5 @@ outPath = self.localOutPath + "/BatchManifest.xml" - tmpPath = "./tmp.xml" + tmpPath = self.localOutPath + "/tmp.xml" self.logger.infoPair("Creating manifest", outPath) root = Element('manifest') @@ -202,5 +241,5 @@ root.attrib['type'] = self.batchType root.attrib['timestamp'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - if self.batchType != "IN": root.attrib['survey'] = self.pspsSurvey + if self.batchType != "IN": root.attrib['survey'] = self.config.pspsSurvey try: self.minObjID except: pass @@ -210,19 +249,8 @@ else: root.attrib['maxObjId'] = str(self.maxObjID) - # get md5sum - p = Popen("md5sum " + self.outputFitsPath, shell=True, stdout=PIPE) - p.wait() - out = p.stdout.read() - md5sum = out[0:out.rfind(" ")] - - # get file size - fileSize = os.path.getsize(self.outputFitsPath) - # file information child = Element('file') root.append(child) child.attrib['name'] = self.outputFitsFile - child.attrib['bytes'] = str(fileSize) - child.attrib['md5'] = md5sum # now create doc and write to file @@ -240,22 +268,34 @@ Publishes this batch to the datastore ''' - def publishToDatastore(self): + def tarAndZip(self): # set up filenams and paths - tarFile = self.batchName + ".tar" - tarPath = self.subDir + "/" + tarFile - - tarballFile = tarFile + ".gz" - tarballPath = self.subDir + "/" + tarballFile + tarFile = Batch.getTarFile(self.batchID) + tarPath = Batch.getTarPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID) + tarballFile = Batch.getTarballFile(self.batchID) + tarballPath = Batch.getTarballPath(self.config.basePath, self.batchType, self.config.dvoLabel, self.batchID) # tar directory - p = Popen("tar -cvf " + tarPath + "\ - -C " + self.subDir + " \ - " + self.batchName, shell=True, stdout=PIPE) + cmd = "tar -cvf " + tarPath + " -C " + self.subDir + " " + self.batchName + self.logger.infoPair("Creating tar archive", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + if p.returncode != 0: + self.logger.errorPair("tar command", "failed") + return False + # zip tar archive - p = Popen("gzip -c " + tarPath + " > " + tarballPath, shell=True, stdout=PIPE) + cmd = "gzip -c " + tarPath + " > " + tarballPath + self.logger.infoPair("Compressing tar archive", cmd) + p = Popen(cmd, shell=True, stdout=PIPE) p.wait() + + if p.returncode != 0: + self.logger.errorPair("gzip command", "failed") + return False + + # only now can we report that the batch has successfully processed + self.ippToPspsDb.updateProcessed(self.batchID, 1) # delete tar file and original directory @@ -263,7 +303,12 @@ shutil.rmtree(self.localOutPath) - # now publish to the datastore - if self.datastore.publish(self.batchName, self.subDir, tarballFile, "tgz"): - self.ippToPspsDb.updateLoadedToDatastore(self.batchID, 1) + return True + + ''' + Static method to publish a batch to the datastore + ''' + @staticmethod + def publishToDatastore(datastore, batchID, batchName, subDir, tarballFile): + datastore.publish(batchName, subDir, tarballFile, "tgz") ''' @@ -293,5 +338,5 @@ rs.close() - self.ippToPspsDb.updateMinMaxObjID(self.batchID, self.minObjID, self.maxObjID) + self.ippToPspsDb.updateDetectionStats(self.batchID, self.minObjID, self.maxObjID, self.totalDetections) self.logger.infoPair("Total detections", "%ld" % self.totalDetections) self.logger.infoPair("Min objID", "%ld" % self.minObjID) @@ -309,5 +354,5 @@ self.tablesToExport.append(table.name) - self.alterPspsTables(); + return self.alterPspsTables(); ''' @@ -329,8 +374,13 @@ Accepts a regular expression filter so not all tables need to be imported ''' - def importIppTables(self, filter=""): + def importIppTables(self, columns="*", filter=""): self.logger.infoPair("Importing tables with filter", filter) - tables = stilts.treads(self.fits.getPath()) + + try: + tables = stilts.treads(self.fits.getPath()) + except: + self.logger.errorPair("STILTS could not import from", self.fits.getPath()) + return False count = 0 @@ -347,4 +397,5 @@ # IPP FITS files are littered with infinities, so remove these self.logger.debug("Removing Infinity values from all columns") + table = stilts.tpipe(table, cmd='keepcols "' + columns + '"') table = stilts.tpipe(table, cmd='replaceval -Infinity null *') table = stilts.tpipe(table, cmd='replaceval Infinity null *') @@ -396,7 +447,7 @@ try: stilts.twrites(_tables, self.outputFitsPath, fmt='fits') - self.ippToPspsDb.updateProcessed(self.batchID, 1) except: self.logger.exception("Could not write to FITS") + self.ippToPspsDb.updateProcessed(self.batchID, -1) return False @@ -432,14 +483,13 @@ def getIDsFromDVO(self): - if self.scratchDb.getRowCount("dvoMeta") < 1: - self.logger.error("No DVO IDs found in dvoMeta") + if self.scratchDb.getRowCount(self.scratchDb.dvoImagesTable) < 1: + self.logger.errorInfo("No DVO images found in", self.scratchDb.dvoImagesTable) return False # TODO path to DVO prog hardcoded temporarily - cmd = "../src/dvograbber " + self.configPath + " " + self.dvoLocation + cmd = "../src/dvograbber " + self.config.settingsPath + " " + self.scratchDb.dbName + " " + self.config.dvoLocation self.logger.infoPair("Running DVO", cmd) p = Popen(cmd, shell=True, stdout=PIPE) p.wait() - # out = p.stdout.read() rowCount = self.scratchDb.getRowCount("dvoDetection") @@ -453,12 +503,6 @@ ''' - Checks whether this batch has already been processed and published. To be implemented by all subclasses - ''' - def alreadyProcessed(self): - self.logger.warn("alreadyProcessed not implemented") - - - ''' Creates and publishes a batch + TODO all methods call below should throw exceptions on failure ''' def run(self): @@ -466,15 +510,27 @@ if not self.everythingOK: self.logger.error("Aborting this batch due to (hopefully) previously reported errors") + self.ippToPspsDb.updateProcessed(self.batchID, -1) return - self.createEmptyPspsTables() + if not self.createEmptyPspsTables(): + self.logger.error("Aborting this batch due to (hopefully) previously reported errors") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + return + self.importIppTables() - if self.populatePspsTables(): - if self.exportPspsTablesToFits(): + if not self.populatePspsTables(): + self.logger.errorPair("Aborting this batch", "unable to populate PSPS tables") + self.ippToPspsDb.updateProcessed(self.batchID, -1) + else: + if not self.exportPspsTablesToFits(): + self.logger.errorPair("Aborting this batch", "unable to export tables to FITS file") + else: self.writeBatchManifest() - if int(self.doc.find("options/publishToDatastore").text): self.publishToDatastore() - if int(self.doc.find("options/reportNulls").text): self.reportNullsInAllPspsTables(False) - - self.logger.infoPair("Batch......", "complete") + if self.config.datastorePublishing: + # tar and zip ready for publication to datastore + if self.tarAndZip(): + # now publish to the datastore + tarballFile = Batch.getTarballFile(self.batchID) + Batch.publishToDatastore(self.datastore, self.batchID, self.batchName, self.subDir, tarballFile) from datastore import Datastore Index: branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/cleanup.py (revision 33415) @@ -6,51 +6,133 @@ import re import sys +import time +import logging.config -import logging.config -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb +from ipptopsps import IppToPsps from batch import Batch +from datastore import Datastore +from dxlayer import DXLayer -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "\n** Usage: " + sys.argv[0] + " \n" - sys.exit(1) +''' +Cleanup class +''' +class Cleanup(IppToPsps): -# open config file -configDoc = ElementTree(file=CONFIG) + ''' + Constructor + ''' + def __init__(self, argv): + super(Cleanup, self).__init__(argv) -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger(sys.argv[0]) -logger.setup(configDoc, sys.argv[0]) - -ippToPspsDb = IppToPspsDb(logger, configDoc) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text - -ids = ippToPspsDb.getMergedButNotDeletedBatchIDs(EPOCH, DVOLABEL) - -logger.infoPair("Merged but un-deleted batches", "%d" % len(ids)) + if len(argv) > 2: self.parsePollTimeArg(sys.argv[2]) + ''' + Overrides base-class version + ''' + def refreshConfig(self): + ret = super(Cleanup, self).refreshConfig() + self.dxlayer = DXLayer(self.logger) + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + return ret -logger.info("+-----------+-------------------------+-------------------------+-------------------------+") -logger.info("| Batch | Removed from datastore | Removed from local disk | Removed from DXLayer |") -logger.info("+-----------+-------------------------+-------------------------+-------------------------+") + ''' + Runs... + ''' + def run(self): -for id in ids: + if self.SECONDS: self.logger.infoPair("Performing clean-up every", "%f hours" % (self.SECONDS/60.0/60.0)) + while True: + + self.config.printDeletionPolicy() + self.clean("P2") + self.clean("ST") + + self.checkClientStatus() + if not self.waitForPollTime(): break + + ''' + Finds stuff to delete for this batch type, then: + + - removes from the datastore + - deletes from local disk + - deletes files from DXLayer + + ''' + def clean(self, batchType): + + self.logger.infoSeparator() + self.logger.infoTitle("Cleaning up batch type %s" % batchType) + + loadedDatastoreIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) + loadedDxlayerIDs = self.ippToPspsDb.getLoadedToODMButNotDeletedFromDXLayer(batchType, self.config.epoch, self.config.dvoLabel) + mergedLocalIDs = self.ippToPspsDb.getMergedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) + + purgedDatastoreIDs = self.ippToPspsDb.getPurgedButNotDeletedFromDatastore(batchType, self.config.epoch, self.config.dvoLabel) + purgedLocalIDs = self.ippToPspsDb.getPurgedButNotDeletedFromLocalDisk(batchType, self.config.epoch, self.config.dvoLabel) + + self.logger.infoPair("Loaded to ODM but still on datastore", "%d" % len(loadedDatastoreIDs)) + self.logger.infoPair("Loaded to ODM but still in DXLayer", "%d" % len(loadedDxlayerIDs)) + self.logger.infoPair("Merged but still on local disk", "%d" % len(mergedLocalIDs)) + self.logger.infoPair("Purged but still on datastore", "%d" % len(purgedDatastoreIDs)) + self.logger.infoPair("Purged but still on local disk", "%d" % len(purgedLocalIDs)) - batchName = Batch.getNameFromID(id) - removedFromDatastore = 0 - removedFromLocalDisk = 0 - removedFromDXLayer = 0 + if self.config.deleteLocal: + self.logger.infoSeparator() + count = 0 + ids = mergedLocalIDs + purgedLocalIDs + for id in ids: + + if Batch.deleteFromDisk(self.logger, self.config.basePath, batchType, self.config.dvoLabel, id): + self.ippToPspsDb.updateDeletedLocal(id, 1) + count = count + 1 + + self.logger.infoSeparator() + self.logger.infoPair("Removed from local disk", "%d" % count) + self.logger.infoPair("Remaining on local disk", "%d" % (len(ids) - count)) + + + if self.config.deleteDatastore: + self.logger.infoSeparator() + count = 0 + ids = loadedDatastoreIDs + purgedDatastoreIDs + for id in ids: + + batchName = Batch.getNameFromID(id) + + if self.datastore.remove(batchName): + count = count + 1 + + self.logger.infoSeparator() + self.logger.infoPair("Removed from datastore", "%d" % count) + self.logger.infoPair("Remaining on datastore", "%d" % (len(ids) - count)) + + if self.config.deleteDxLayer: + self.logger.infoSeparator() + count = 0 + for id in loadedDxlayerIDs: + + if self.dxlayer.deleteBatch(id): + self.ippToPspsDb.updateDeletedDxlayer(id, 1) + count = count + 1 + + self.logger.infoSeparator() + self.logger.infoPair("Removed from DXLayer", "%d" % count) + self.logger.infoPair("Remaining on DXLayer", "%d" % (len(loadedDxlayerIDs) - count)) + - logger.info("| %9s | %23d | %23d | %23d |", - batchName, - removedFromDatastore, - removedFromLocalDisk, - removedFromDXLayer) + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Cleanup, self).printUsage() + print " []" -logger.info("+-----------+-------------------------+-------------------------+-------------------------+") + +''' +Start of program +''' +cleanup = Cleanup(sys.argv) +cleanup.run() +cleanup.exitProgram("finished") Index: branches/meh_branches/ppstack_test/ippToPsps/jython/config.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/config.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/config.py (revision 33415) @@ -0,0 +1,131 @@ +#!/usr/bin/env jython + +import logging +from xml.etree.ElementTree import ElementTree, Element, tostring + +from pslogger import PSLogger + + +''' +A class encapsulating the ippToPsps config file +''' +class Config(object): + + + ''' + Constructor + + Basically reads the entire config and stores values to class variables + ''' + def __init__(self, programName, name): + + self.programName = programName + self.name = name + self.settingsPath = "../config/settings.xml" # TODO + self.logger = None + + self.settingsDoc = ElementTree(file=self.settingsPath) + self.logPath = self.settingsDoc.find("logPath").text + + + # this is the border (in degrees) that we place around any loading box of + # PS1 pointings to ensure we pull a large enough area out of DVO + self.BORDER = 1.6 + self.isLoaded = False + + ''' + Prints everything for this config + ''' + def printAll(self): + + try: + self.logger.infoSeparator() + self.logger.infoPair("Config name", self.name) + self.logger.infoPair("Survey", self.survey) + self.logger.infoPair("Publishing to PSPS as survey", self.pspsSurvey) + self.logger.infoPair("Loading epoch", self.epoch) + self.logger.infoPair("Data release", "%d" % self.dataRelease) + self.logger.infoBool("Test mode?", self.test) + self.logger.infoBool("Forcing?", self.force) + self.logger.infoBool("Reporting NULLS?", self.reportNulls) + for batchType in self.batchTypes: self.logger.infoPair("Queuing batch type", batchType) + except: + pass + + self.printDvoInfo() + self.printDatastoreInfo() + self.printBoxCoords() + self.printDeletionPolicy() + try: self.logger.infoSeparator() + except: pass + + + ''' + Creates a logger object and returns it + ''' + def getLogger(self, host, pid, stdout=1, sendToFile=0): + + logging.setLoggerClass(PSLogger) + self.logger = logging.getLogger(self.programName) + self.logger.setup(self.programName, self.logPath, self.name, host, pid, stdout, sendToFile) + + return self.logger + + ''' + Prints the currently set DVO info + ''' + def printDvoInfo(self): + + try: + self.logger.infoPair("DVO label", self.dvoLabel) + self.logger.infoPair("DVO location", self.dvoLocation) + except: + pass + + ''' + Prints datastore info + ''' + def printDatastoreInfo(self): + + try: + self.logger.infoBool("Datastore publishing?", self.datastorePublishing) + if self.datastorePublishing: + self.logger.infoPair("Datastore type", self.datastoreProduct) + self.logger.infoPair("Datastore product", self.datastoreType) + except: + pass + + ''' + Prints the current deletion policy + ''' + def printDeletionPolicy(self): + + try: + self.logger.infoBool("Deleting local?", self.deleteLocal) + self.logger.infoBool("Deleting datastore?", self.deleteDatastore) + self.logger.infoBool("Deleting DXLayer?", self.deleteDxLayer) + except: + pass + + ''' + Prints the currently set RA/Dec bounding box + ''' + def printBoxCoords(self): + + try: + self.logger.infoPair("RA limits", "%.1f -> %.1f" % (self.minRa, self.maxRa)) + self.logger.infoPair("Dec limits", "%.1f -> %.1f" % (self.minDec, self.maxDec)) + self.logger.infoPair("Loading box size", "%.1f degrees" % self.boxSize) + except: + pass + + + ''' + Various db metadata getters + ''' + def getDbName(self, dbType): return self.settingsDoc.find(dbType +"/name").text + def getDbHost(self, dbType): return self.settingsDoc.find(dbType +"/host").text + def getDbUser(self, dbType): return self.settingsDoc.find(dbType +"/user").text + def getDbPassword(self, dbType): return self.settingsDoc.find(dbType +"/password").text + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/init.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/init.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/init.xml (revision 33415) @@ -0,0 +1,75 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + BOGUS + BOGUS + + + + + 0 + BOGUS + BOGUS + + + + /data/ipp005.0/rhenders + + + 0 + + + + 2011-07-01-15:34 + 3PI + 3PI + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110621.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110621.xml (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110621.xml (revision 33415) @@ -49,11 +49,18 @@ + + + 1 + 1 + 1 + + 2011-07-01-15:34 3PI 3PI - 0 + 1 1 - 1 + 0 1 0 Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110809.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110809.xml (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110809.xml (revision 33415) @@ -7,5 +7,5 @@ - ipptopsps_scratch3 + ipptopsps_scratch localhost ipp @@ -29,7 +29,15 @@ + + + czardb + ippdb01 + ipp + ipp + + - PSPS_test + PSPS_JHU IPP_PSPS @@ -49,13 +57,20 @@ + + + 1 + 1 + 1 + + 2011-08-16 3PI 3PI - 0 - 1 - 1 - 1 - 0 + 1 + 0 + 0 + 0 + 1 0 + + + + + + ipptopsps_scratch2 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + LAP.ThreePi.20110809 + /data/ipp005.0/gpc1/catdirs/LAP.ThreePi.20110809 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-08-16 + 3PI + 3PI + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110809_3.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110809_3.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/lap20110809_3.xml (revision 33415) @@ -0,0 +1,82 @@ + + + + + + + + + ipptopsps_scratch3 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + LAP.ThreePi.20110809 + /data/ipp005.0/gpc1/catdirs/LAP.ThreePi.20110809 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-08-16 + 3PI + 3PI + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/newmd4.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/newmd4.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/newmd4.xml (revision 33415) @@ -0,0 +1,90 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 1 + MD04.20111128 + /data/ipp005.0/gpc1/catdirs/MD04.20111128 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 1 + 1 + 1 + + + + 2011-12-03 + MD04 + MDF + 1 + 1 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/newsas.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/newsas.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/newsas.xml (revision 33415) @@ -0,0 +1,90 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + SAS.20111201 + /data/ipp005.0/gpc1/catdirs/SAS.20111201 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 1 + 1 + 1 + + + + 2011-12-01 + 3PI + SA3 + 1 + 1 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi.xml (revision 33415) @@ -0,0 +1,94 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -30 + 90 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi2.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi2.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi2.xml (revision 33415) @@ -0,0 +1,86 @@ + + + + + + + + + ipptopsps_scratch2 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -100 + 100 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi3.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi3.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepi3.xml (revision 33415) @@ -0,0 +1,86 @@ + + + + + + + + + ipptopsps_scratch3 + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -100 + 100 + + + + /data/ipp005.0/rhenders + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 0 + 0 + 1 + 0 + + + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepiTest.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepiTest.xml (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/oldthreepiTest.xml (revision 33415) @@ -0,0 +1,95 @@ + + + + + + + + + ipptopsps_scratch + localhost + ipp + ipp + + + + + gpc1 + ippdb01 + ipp + ipp + + + + + ipptopsps_test + ippdb01 + ipp + ipp + + + + + czardb + ippdb01 + ipp + ipp + + + + + PSPS_JHU + IPP_PSPS + + + + + 0 + ThreePi.V3 + /data/ipp005.0/gpc1/catdirs/ThreePi.V3 + 90 + 270 + -30 + 90 + + + + /data/ipp005.0/rhenders/test + + + 0 + + + + + 0 + 1 + 1 + + + + 2011-10-27 + 3PI + OLD + 1 + 0 + 1 + 0 + 0 + 0 + + + 72203 + + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sas12.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sas12.xml (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sas12.xml (revision 33415) @@ -49,4 +49,11 @@ + + + 1 + 1 + 1 + + 2011-08-09 Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sas123.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sas123.xml (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sas123.xml (revision 33415) @@ -49,4 +49,11 @@ + + + 1 + 1 + 1 + + 2011-08-09 Index: branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sasFootprint.xml =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sasFootprint.xml (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/configs/sasFootprint.xml (revision 33415) @@ -49,4 +49,11 @@ + + + 1 + 1 + 1 + + 2011-08-09 Index: branches/meh_branches/ppstack_test/ippToPsps/jython/czardb.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/czardb.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/czardb.py (revision 33415) @@ -0,0 +1,50 @@ +#!/usr/bin/env jython + +import re +import sys +import os +import logging + +from mysql import MySql +from java.sql import * + +''' +Class for CzarDb database connectivity +''' +class CzarDb(MySql): + + ''' + Constructor + ''' + def __init__(self, logger, config): + super(CzarDb, self).__init__(logger, config, "czardatabase") + + + ''' + Updates a stage table with pending, processed, faults and label + ''' + def insertStats(self, stage, label, batchType, pending, processed, faults): + + sql = "INSERT INTO " + stage + " \ + (label, pending, processed, faults) \ + VALUES ('" + label + "_" + batchType + "', \ + " + str(pending) + ", \ + " + str(processed) + ", \ + " + str(faults) + ")" + + try: + self.execute(sql) + except: + pass + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("CzarDb Desstructor") + self.stmt.close() + self.con.close() + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/datastore.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/datastore.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/datastore.py (revision 33415) @@ -5,5 +5,8 @@ import logging import sys +import re from xml.etree.ElementTree import ElementTree, Element, tostring + +from ipptopspsdb import IppToPspsDb ''' @@ -16,16 +19,10 @@ ''' - def __init__(self, logger, doc): + def __init__(self, logger, config, ippToPspsDb): # setup logger self.logger = logger - self.doc = doc - self.logger.debug("Datastore constructor") - - # open config - self.product = doc.find("datastore/product").text - self.type = doc.find("datastore/type").text - - self.logger.debug("Using product: '" + self.product + "' and type: '" + self.type + "'") + self.config = config + self.ippToPspsDb = ippToPspsDb ''' @@ -34,4 +31,7 @@ def publish(self, name, path, file, fileType): + # TODO should handle this better + batchID = int(name[1:]) + self.logger.infoPair("Attempting to publish", path + "/" + file) tempFile = tempfile.NamedTemporaryFile(mode='w+b') @@ -39,9 +39,9 @@ tempFile.seek(0) - # build dsreg command command + # build dsreg command command = "dsreg --add " + name + "\ - --copy --datapath " + path + "\ - --type " + self.type + "\ - --product " + self.product + "\ + --link --datapath " + path + "\ + --type " + self.config.datastoreType + "\ + --product " + self.config.datastoreProduct + "\ --list " + tempFile.name @@ -52,32 +52,45 @@ self.logger.errorPair("Datastore publish", "failed") ret = False + self.ippToPspsDb.updateLoadedToDatastore(batchID, -1) else: self.logger.infoPair("Datastore publish", "successful") ret = True + self.ippToPspsDb.updateLoadedToDatastore(batchID, 1) + tempFile.close() - tempFile.close() return ret - ''' Removes an item from the datastore + + NB this will return a success if the batch is not found on the datastore ''' def remove(self, name): + + # TODO should handle this better + batchID = int(name[1:]) command = "dsreg \ --del " + name + " \ --rm \ - --product " + self.product + --product " + self.config.datastoreProduct - p = Popen(command, shell=True, stdout=PIPE) + p = Popen(command, shell=True, stdout=PIPE, stderr=PIPE) p.wait() + output, errors = p.communicate() if p.returncode != 0: - self.logger.errorPair("Datastore removal failed", name) - ret = False + if re.search("not found in", errors): + self.logger.infoPair("Datastore batch not found", name) + ret = True + self.ippToPspsDb.updateDeletedDatastore(batchID, 1) + else: + self.logger.infoPair("Datastore removal FAILED", name) + ret = False else: + self.logger.infoPair("Datastore removal successful", name) ret = True - self.logger.infoPair("Datastore removal successful", name) + self.ippToPspsDb.updateDeletedDatastore(batchID, 1) return ret Index: branches/meh_branches/ppstack_test/ippToPsps/jython/datastoreDupeChecker.sh =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/datastoreDupeChecker.sh (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/datastoreDupeChecker.sh (revision 33415) @@ -0,0 +1,11 @@ +rm index.txt +wget http://datastore.ipp.ifa.hawaii.edu/PSPS_JHU/index.txt + +echo "Dupes:" +echo "" + +more index.txt | awk '{print $1}' | uniq -d +echo "" +echo "Done" + +#wget http://datastore.ipp.ifa.hawaii.edu/PSPS_JHU/B00263471/index.txt Index: branches/meh_branches/ppstack_test/ippToPsps/jython/detectionbatch.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/detectionbatch.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/detectionbatch.py (revision 33415) @@ -22,4 +22,7 @@ ''' DetectionBatch class + +This class, a sub-class of Batch, processes single exposures in the form of IPP smf files + ''' class DetectionBatch(Batch): @@ -30,19 +33,23 @@ def __init__(self, logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, - camID): + scratchDb, + camID, + batchID, + useFullTables): super(DetectionBatch, self).__init__( logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, camID, + batchID, "P2", - gpc1Db.getCameraStageSmf(camID)) + gpc1Db.getCameraStageSmf(camID), + useFullTables) if not self.everythingOK: return @@ -50,4 +57,8 @@ # get camera meta data meta = self.gpc1Db.getCameraStageMeta(self.id) + if not meta: + self.everythingOK = False + return + self.expID = meta[0]; self.expName = meta[1]; @@ -64,5 +75,5 @@ # if test mode - if self.testMode: + if self.config.test: self.startX = 3 self.endX = 4 @@ -101,4 +112,5 @@ self.filter = self.header['FILTERID'][0:1] + self.filterID = self.scratchDb.getFilterID(self.filter) # insert what we know about this stack batch into the stack table @@ -209,5 +221,5 @@ self.scratchDb.updateFilterID("FrameMeta", self.filter) self.scratchDb.updateAllRows("FrameMeta", "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("FrameMeta", "dataRelease", str(self.config.dataRelease)) ''' @@ -312,5 +324,5 @@ ," + self.safeDictionaryAccess(header, 'DAPMIFIT') + " \ ,'" + self.safeDictionaryAccess(header, 'DETECTOR') + "' \ - ," + str(self.scratchDb.getDvoImageFlags(header['SOURCEID'], header['IMAGEID'])) + " \ + ," + str(self.scratchDb.getDvoImageFlags(header['IMAGEID'])) + " \ ,'" + self.safeDictionaryAccess(header, 'DETREND.MASK') + "' \ ,'" + self.safeDictionaryAccess(header, 'DETREND.DARK') + "' \ @@ -356,5 +368,5 @@ self.scratchDb.updateFilterID(tableName, self.filter) self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) if 'NASTRO' in header: self.totalNumPhotoRef = self.totalNumPhotoRef + int(header['NASTRO']) self.scratchDb.replaceNullsInThisColumn(tableName, "polyOrder", "0") @@ -367,5 +379,7 @@ pspsTableName = "Detection_" + ota ippTableName = ota + "_psf" - + + results['ORIGINALTOTAL'] = self.scratchDb.getRowCount(ippTableName) + # drop then re-create table self.scratchDb.dropTable(pspsTableName) @@ -385,4 +399,7 @@ sql = "INSERT IGNORE INTO " + pspsTableName + " ( \ ippDetectID \ + ,filterID \ + ,surveyID \ + ,obsTime \ ,xPos \ ,yPos \ @@ -395,4 +412,5 @@ ,psfWidMinor \ ,psfTheta \ + ,psfLikelihood \ ,psfCf \ ,momentXX \ @@ -400,13 +418,18 @@ ,momentYY \ ,apMag \ - ,kronFlux \ - ,kronFluxErr \ ,infoFlag \ ,sky \ ,skyErr \ ,sgSep \ + ,activeFlag \ + ,assocDate \ + ,historyModNum \ + ,dataRelease \ ) \ SELECT \ IPP_IDET \ + , " + str(self.filterID) + "\ + , " + str(self.surveyID) + " \ + ," + str(self.obsTime) + " \ ,X_PSF \ ,Y_PSF \ @@ -419,4 +442,5 @@ ,PSF_MINOR \ ,PSF_THETA \ + ,psfLikelihood(EXT_NSIGMA) \ ,PSF_QF \ ,MOMENTS_XX \ @@ -424,23 +448,26 @@ ,MOMENTS_YY \ ,AP_MAG \ - ,KRON_FLUX \ - ,KRON_FLUX_ERR \ ,FLAGS\ ,SKY \ ,SKY_SIGMA \ ,EXT_NSIGMA \ + , 0 \ + , '" + self.dateStr + "' \ + , 0 \ + , " + str(self.config.dataRelease) + "\ FROM " + ippTableName self.scratchDb.execute(sql) - # set obsTime - sql = "UPDATE " + pspsTableName + " SET obsTime = %f, assocDate = '%s', activeFlag = 0" % (self.obsTime, self.dateStr) - self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(pspsTableName, "dataRelease", str(self.dataRelease)) - self.scratchDb.updateAllRows(pspsTableName, "historyModNum", "0") - - self.scratchDb.updateAllRows(pspsTableName, "surveyID", str(self.surveyID)) - self.scratchDb.updateFilterID(pspsTableName, self.filter) - # now delete bad flux and bad chip positions + sql="DELETE FROM " + pspsTableName + " WHERE instFlux < 0.0000001" # TODO clearly a hack = 0 not allowed for instFlux + self.scratchDb.execute(sql) + + # update cosmic ray and extended likelihoods + sql="UPDATE " + pspsTableName + " SET extendedLikelihood = 0, crLikelihood = 1.0 - psfLikelihood WHERE sgSep <= 0" + self.scratchDb.execute(sql) + sql="UPDATE " + pspsTableName + " SET crLikelihood = 0, extendedLikelihood = 1.0 - psfLikelihood WHERE sgSep > 0" + self.scratchDb.execute(sql) + + # remove detections will NULL inst flux or NULL peak ADU results['NULLINSTFLUX'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "instFlux") results['NULLPEAKADU'] = self.scratchDb.reportAndDeleteRowsWithNULLS(pspsTableName, "peakADU") @@ -472,5 +499,5 @@ self.scratchDb.execute(sql) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) ''' @@ -501,5 +528,5 @@ self.scratchDb.updateAllRows(tableName, "calibModNum", str(self.calibModNum)) - self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows(tableName, "dataRelease", str(self.config.dataRelease)) @@ -514,82 +541,8 @@ self.scratchDb.makeColumnPrimaryKey("Detection", "ippDetectID") - ''' - Applies indexes to the IPP tables - ''' - def indexIppTables(self): - - self.logger.infoPair("Creating indexes on", "IPP tables") - - for x in range(self.startX, self.endX): - for y in range(self.startY, self.endY): - - # dodge the corners - if x==0 and y==0: continue - if x==0 and y==7: continue - if x==7 and y==0: continue - if x==7 and y==7: continue - - extension = "XY%d%d_psf" % (x, y) - self.scratchDb.createIndex(extension, "IPP_IDET") - - ''' - Updates provided table with DVO IDs from DVO table - ''' - def updateDvoIDs(self, table, sourceID, externID): - - imageID = self.scratchDb.getImageIDFromExternID(sourceID, externID) - self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) - sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ - a.ippObjID = b.ippObjID, \ - a.detectID = b.detectID, \ - a.objID = b.objID, \ - a.infoFlag = b.flags << 32 | a.infoFlag \ - WHERE a.ippDetectID = b.ippDetectID \ - AND b.sourceID = " + str(sourceID) + " \ - AND b.imageID = " + str(imageID) - - self.scratchDb.execute(sql) - - ''' - Generates psf, cosmic ray and extended source likelihoods - ''' - def populateLikelihoods(self, tableName): - - sql = "SELECT ippDetectID, psfLikelihood, crLikelihood, extendedLikelihood, sgSep FROM " + tableName - rs = self.scratchDb.executeUpdatableQuery(sql) - - sqrt2 = Math.sqrt(2) - - while rs.next(): - - sgSep = rs.getDouble(5) - psfLikelihood = Erf.erfc(Math.abs(sgSep)/sqrt2) - - if sgSep > 0: - crLikelihood = 0 - extendedLikelihood = 1.0 - psfLikelihood - else: - crLikelihood = 1.0 - psfLikelihood - extendedLikelihood = 0 - - # update columns - rs.updateDouble(2, psfLikelihood ) - rs.updateDouble(3, crLikelihood ) - rs.updateDouble(4, extendedLikelihood ) - - # now 'commit' to database - rs.updateRow(); - - - ''' - Does the processing, i.e. pulling stuff from IPP tables into PSPS tables - ''' - def populatePspsTables(self): - self.populateFrameMeta() - # dictionary objects to hold sourceIDs and imageIDs for later - sourceIDs = {} - imageIDs = {} + # dictionary objects to hold imageIDs for later + self.imageIDs = {} # loop through all OTAs and populate ImageMeta extensions @@ -614,8 +567,6 @@ # check we have valid sourceID/imageID pair from the header - if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": - continue - if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": - continue + if self.safeDictionaryAccess(header, 'SOURCEID') == "NULL": continue + if self.safeDictionaryAccess(header, 'IMAGEID') == "NULL": continue # store sourceID/imageID combo in Db so DVO can look up later @@ -624,6 +575,5 @@ # store these for later - sourceIDs[ota] = header['SOURCEID'] - imageIDs[ota] = header['IMAGEID'] + self.imageIDs[ota] = header['IMAGEID'] # populate ImageMeta @@ -632,5 +582,51 @@ # now run DVO code to get all IDs - if not self.useFullTables: self.getIDsFromDVO() + if not self.useFullTables: + if not self.getIDsFromDVO(): return False + + return True + + + ''' + Applies indexes to the IPP tables + ''' + def indexIppTables(self): + + self.logger.infoPair("Creating indexes on", "IPP tables") + + for x in range(self.startX, self.endX): + for y in range(self.startY, self.endY): + + # dodge the corners + if x==0 and y==0: continue + if x==0 and y==7: continue + if x==7 and y==0: continue + if x==7 and y==7: continue + + extension = "XY%d%d_psf" % (x, y) + self.scratchDb.createIndex(extension, "IPP_IDET") + + ''' + Updates provided table with DVO IDs from DVO table + ''' + def updateDvoIDs(self, table, externID): + + imageID = self.scratchDb.getImageIDFromExternID(externID) + self.logger.debug("Updating table '" + table + "' with DVO IDs using imageID = %d" % imageID) + sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ + a.ippObjID = b.ippObjID, \ + a.detectID = b.detectID, \ + a.objID = b.objID, \ + a.infoFlag = b.flags << 32 | a.infoFlag \ + WHERE a.ippDetectID = b.ippDetectID \ + AND b.imageID = " + str(imageID) + + self.scratchDb.execute(sql) + + + ''' + Does the processing, i.e. pulling stuff from IPP tables into PSPS tables + ''' + def populatePspsTables(self): # loop through all OTAs again to update with DVO IDs @@ -640,7 +636,8 @@ otaCount = 0 results = {} - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") - self.logger.info("| OTA | Sat Det | NULL instFlux | NULL peak ADU | NULL obj ID | Remainder |") - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") + totalOriginal = totalSatDet = totalNulIInstFlux = totalNullPeakFlux = totalNullObjID = totalDetections = 0 + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("| OTA | Initial total | Sat Det | NULL instFlux | NULL peak ADU | NULL obj ID | Remainder |") + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") for x in range(self.startX, self.endX): for y in range(self.startY, self.endY): @@ -653,7 +650,10 @@ ota = "XY%d%d" % (x, y) - if ota not in sourceIDs: continue + if ota not in self.imageIDs: continue #self.logger.infoTitle("Processing " + ota) + if not self.scratchDb.astrometricSolutionOK(ota): + self.logger.info("| %5s | ------------- Bad astrometric solution : rejecting ------------ |" % ota) + continue # populate remainder of tables @@ -661,11 +661,11 @@ # now add DVO IDs - self.updateDvoIDs("Detection_" + ota, sourceIDs[ota], imageIDs[ota]) + self.updateDvoIDs("Detection_" + ota, self.imageIDs[ota]) results['NULLOBJID'] = self.scratchDb.reportAndDeleteRowsWithNULLS("Detection_" + ota, "objID") self.updateImageID("Detection_" + ota, x, y) - self.populateLikelihoods("Detection_" + ota) rowCount = self.scratchDb.getRowCount("Detection_" + ota) - self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d |", + self.logger.info("| %5s | %13d | %13d | %13d | %13d | %13d | %13d |", ota, + results['ORIGINALTOTAL'], results['SATDET'], results['NULLINSTFLUX'], @@ -674,4 +674,11 @@ rowCount) + totalOriginal = totalOriginal + results['ORIGINALTOTAL'] + totalSatDet = totalSatDet + results['SATDET'] + totalNulIInstFlux = totalNulIInstFlux + results['NULLINSTFLUX'] + totalNullPeakFlux = totalNullPeakFlux + results['NULLPEAKADU'] + totalNullObjID = totalNullObjID + results['NULLOBJID'] + totalDetections = totalDetections + rowCount + # check we have something in this Detection table TODO add this to table above if rowCount < 1: @@ -681,5 +688,5 @@ # update ImageMeta with count of detections for this OTA and photoCodeID sql = "UPDATE ImageMeta_" + ota + " \ - SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(sourceIDs[ota], imageIDs[ota])) + SET nDetect = %d, photoCalID = %d" % (self.scratchDb.getRowCount("Detection_" + ota), self.scratchDb.getPhotoCalID(self.imageIDs[ota])) self.scratchDb.execute(sql) @@ -696,6 +703,14 @@ otaCount = otaCount + 1 - # TODO print totals here - self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+") + # print totals + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") + self.logger.info("| Total | %13d | %13d | %13d | %13d | %13d | %13d |", + totalOriginal, + totalSatDet, + totalNulIInstFlux, + totalNullPeakFlux, + totalNullObjID, + totalDetections) + self.logger.info("+-------+---------------+---------------+---------------+---------------+---------------+---------------+") # if we only have one table export, i.e. FrameMeta, then get out of here @@ -737,8 +752,10 @@ def importIppTables(self, filter=""): - if self.testMode: regex = "XY33.psf" + if self.config.test: regex = "XY33.psf" else : regex = ".*.psf" - - return super(DetectionBatch, self).importIppTables(regex) + + columns = "IPP_IDET X_PSF Y_PSF X_PSF_SIG Y_PSF_SIG PSF_INST_MAG PSF_INST_MAG_SIG PEAK_FLUX_AS_MAG PSF_MAJOR PSF_MINOR PSF_THETA EXT_NSIGMA PSF_QF MOMENTS_XX MOMENTS_XY MOMENTS_YY AP_MAG FLAGS SKY SKY_SIGMA EXT_NSIGMA" + + return super(DetectionBatch, self).importIppTables(columns, regex) Index: branches/meh_branches/ppstack_test/ippToPsps/jython/dvo.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/dvo.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/dvo.py (revision 33415) @@ -0,0 +1,516 @@ +#!/usr/bin/env jython + +import stilts +import datetime +import re +import sys +import os +import logging +import glob +from subprocess import call, PIPE, Popen + +from pslogger import PSLogger +from scratchdb import ScratchDb + +import time +from java.lang import * +from java.sql import * + + +''' +A class that encapsulates a DVO database + +Pulls relavant bits and pieces into the scratch MySQL database and then provides access queries +''' +class Dvo(object): + + ''' + Constructor + + - sets up logging + - checks we are using the correct DVO + - checks we have up-to-date Images.dat + - checks we have up-to-date SkyTable.fits + + ''' + def __init__(self, logger, config): + + # set up logging + self.logger = logger + self.config = config + + # set up empty lists + self.regionsToIngest = [] + self.regionsToPurge = [] + self.allPopulatedRegions = [] + self.regionsAlreadyIngested = [] + self.regionsIngestedButOutOfDate = [] + + #self.correctDvo = False + + + ''' + Sets to scratch database + ''' + def setScratchDb(self, dbName): + + self.scratchDb = ScratchDb(self.logger, self.config, dbName) + self.scratchDb.setUseFullTables(True) + + # or decide if we are using the right DVO + self.correctDvo = self.scratchDb.isCorrectDvo(self.config.dvoLocation) + + if not self.correctDvo: + print "*******************************************************************************" + response = raw_input("**** Wrong or no DVO in use. Do you want to reset and use '" + self.config.dvoLocation + "' (y/n)? ") + if response == "y": + response = raw_input("**** Are you ABSOLUTELY sure you want to do this? (y/n)? ") + if response == "y": + self.resetAllTables() + + + ''' + Destructor + ''' + def __del__(self): + self.logger.debug("Dvo destructor") + + ''' + Check we are importing the DVO currenly ingested + ''' + def isThisDvoCurrentlyIngested(self): + return self.correctDvo + + ''' + Resets all detections. Be careful + ''' + def resetDetections(self): + + self.logger.infoPair("Resetting", "just detections") + self.scratchDb.resetDvoDetectionTables() + + ''' + Resets all dvo tables. Be very careful.... + ''' + def resetAllTables(self): + + self.logger.infoPair("Resetting", "all tables") + self.scratchDb.resetAllDvoTables() + self.correctDvo = True + + ''' + Loads the DVO Images.dat FITS file + ''' + def loadImages(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return + + # check if we have up-to-date version + path = self.config.dvoLocation + "/Images.dat" + if self.scratchDb.alreadyImportedThisDvoTable(path): + self.logger.debugPair("DVO Images.dat file", "up-to-date") + return + + self.logger.infoSeparator() + self.logger.infoPair("DVO Images.dat file", "NOT up-to-date") + + # first, delete old dvoMeta table + #self.logger.infoPair("Deleting from table", self.scratchDb.dvoMetaTable) + #sql = "DELETE FROM " + self.scratchDb.dvoMetaTable + #self.scratchDb.execute(sql) + #self.scratchDb.optimizeTable(self.scratchDb.dvoMetaTable) + + self.importFits( + path, + "SOURCE_ID IMAGE_ID EXTERN_ID FLAGS PHOTCODE", + self.scratchDb.dvoImagesTable) + self.logger.infoPair("Adding primary key to", self.scratchDb.dvoImagesTable) + self.scratchDb.execute("ALTER TABLE " + self.scratchDb.dvoImagesTable + " ADD PRIMARY KEY (IMAGE_ID, EXTERN_ID)") + + # insert into dvoMetaFull + # NB what we and smf files call IMAGE_ID, DVO calls EXTERN_ID. We are sticking + # with the smf name + ''' + self.logger.infoPair("Populating", "all image meta data") + sql = "INSERT INTO " + self.scratchDb.dvoMetaTable + " ( \ + imageID, \ + externID, \ + flags, \ + photcode \ + ) SELECT \ + IMAGE_ID, \ + EXTERN_ID, \ + FLAGS, \ + PHOTCODE \ + FROM " + self.scratchDb.dvoImagesTable + + try: + self.scratchDb.execute(sql) + except: + self.logger.errorPair("Could not insert into", self.scratchDb.dvoMetaTable) + return + ''' + + self.scratchDb.setImportedThisDvoTable(path) + + ''' + Loads the DVO SkyTable into MySQL and then applies an index + ''' + def loadSkyTable(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return + + path = self.config.dvoLocation + "/SkyTable.fits" + if self.scratchDb.alreadyImportedThisDvoTable(path): + self.logger.debugPair("DVO SkyTable.fits file", "up-to-date") + return + + self.logger.infoSeparator() + self.logger.infoPair("DVO SkyTable.fits file", "NOT up-to-date") + self.importFits( + path, + "R_MIN R_MAX D_MIN D_MAX INDEX NAME", + self.scratchDb.dvoSkyTable) + + self.logger.infoPair("Adding index to", self.scratchDb.dvoSkyTable) + + self.scratchDb.createIndex(self.scratchDb.dvoSkyTable, "INDEX") + + self.scratchDb.setImportedThisDvoTable(path) + self.logger.infoPair("Finished importing SkyTable at", self.config.dvoLocation) + + ''' + Sets a bunch of region files from DVO for a RA/Dec box centered with these coords and with this side length + ''' + def setSkyAreaAsBox(self, ra=180., dec=0., side=181.0): + + halfSide = side/2.0 + self.setSkyArea(ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide) + + ''' + Sets a bunch of region files from DVO for this RA/Dec box + ''' + def setSkyArea(self, minRa=-1., maxRa=361., minDec=-91., maxDec=91): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return + + if (maxRa - minRa <= 0) or (maxDec - minDec) <= 0: + self.logger.errorPair("Zero range in either RA or Dec", + "RA: %.2f -> %.2f, Dec: %.2f -> %.2f" % (minRa, maxRa, minDec, maxDec)) + return False + + # update the two meta tables, Images and SkyTable + #self.loadImages() dont need a new Images table at this point + self.loadSkyTable() + + # set bounding-box coords + self.minRa = minRa + self.maxRa = maxRa + self.minDec = minDec + self.maxDec = maxDec + + # reset all lists + allRegions = self.scratchDb.getDvoRegions(minRa, maxRa, minDec, maxDec) + allIngestedRegions = self.scratchDb.getIngestedDvoRegions() + self.regionsToIngest = [] + self.regionsToPurge = [] + self.allPopulatedRegions = [] + self.regionsAlreadyIngested = [] + self.regionsIngestedButOutOfDate = [] + + # loop through all regions in DVO inside this bounding-box + for region in allRegions: + + cpmPath = self.config.dvoLocation + "/" + region + ".cpm" + cptPath = self.config.dvoLocation + "/" + region + ".cpt" + + # check for existence of cpm and cpt files + if not os.path.isfile(cpmPath): continue + if not os.path.isfile(cptPath): continue + + # ok, have a real, populated file so add it to the list + self.allPopulatedRegions.append(region) + + # if we have already imported up-to-date versions of both the cpm and cpt then we skip this region + if self.scratchDb.alreadyImportedThisDvoTable(cpmPath) and self.scratchDb.alreadyImportedThisDvoTable(cptPath): + self.regionsAlreadyIngested.append(region) + continue + + # check if we have an out-of-date version of either region file, if so, add it to the purge list + if self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cpmPath) or self.scratchDb.haveOutOfDateVersionOfThisDvoTable(cptPath): + self.regionsIngestedButOutOfDate.append(region) + + self.regionsToIngest.append(region) + + # add out-of-range regions for purge list + items = list(set(allIngestedRegions) - set(self.allPopulatedRegions)) + for item in items: self.regionsToPurge.append(item) + + # now add out-of-date items to purge list + for item in self.regionsIngestedButOutOfDate: self.regionsToPurge.append(item) + + self.sizeOfAllPopulatedRegions = self.getDiskSizeOfDvoRegions(self.allPopulatedRegions) + self.sizeOfRegionsAlreadyIngested = self.getDiskSizeOfDvoRegions(self.regionsAlreadyIngested) + self.sizeOfRegionsIngestedButOutOfDate = self.getDiskSizeOfDvoRegions(self.regionsIngestedButOutOfDate) + self.sizeOfRegionsToPurge = self.getDiskSizeOfDvoRegions(self.regionsToPurge) + self.sizeOfRegionsToIngest = self.getDiskSizeOfDvoRegions(self.regionsToIngest) + + ''' + Prints out a summary of the currently set regions + ''' + def printSummary(self): + + self.logger.infoTitle("Summary for currently set DVO region") + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: + self.logger.infoPair("Wrong DVO in use", self.config.dvoLocation) + return + + self.logger.infoPair("RA range", "%.2f -> %.2f" % (self.minRa, self.maxRa)) + self.logger.infoPair("Dec range", "%.2f -> %.2f" % (self.minDec, self.maxDec)) + self.logger.infoPair("Total regions", "%d (%s)" % (len(self.allPopulatedRegions), self.getFileSizeStr(self.sizeOfAllPopulatedRegions))) + self.logger.infoPair("Up-to-date regions already ingested", "%d (%s)" % (len(self.regionsAlreadyIngested), self.getFileSizeStr(self.sizeOfRegionsAlreadyIngested))) + self.logger.infoPair("Out-of-date regions ingested", "%d (%s)" % (len(self.regionsIngestedButOutOfDate), self.getFileSizeStr(self.sizeOfRegionsIngestedButOutOfDate))) + self.logger.infoPair("Total regions to purge", "%d (%s)" % (len(self.regionsToPurge), self.getFileSizeStr(self.sizeOfRegionsToPurge))) + self.logger.infoPair("Total regions to ingest", "%d (%s)" % (len(self.regionsToIngest), self.getFileSizeStr(self.sizeOfRegionsToIngest))) + + + ''' + Returns the size on disk of the regions that need to be ingested in GB + ''' + def getDiskSizeOfRegionsToBeIngested(self): + + try: + return self.sizeOfRegionsToIngest/1073741824.0 + except: + return 0.0 + + ''' + Purges the provided regions from the database + ''' + def purgeRegions(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return + + if len(self.regionsToPurge) < 1: return + + self.logger.infoPair("Purging", "%d regions" % len(self.regionsToPurge)) + deleteCount = self.scratchDb.purgeTheseDvoRegions(self.regionsToPurge) + self.logger.infoPair("Deleted", "%d detections" % deleteCount) + + ''' + Formats a nice file size string + Accepts a size in bytes + ''' + def getFileSizeStr(self, byteSize): + + # format a string for the user -smaller than one MB + if byteSize < 1048576: return "%.1f K" % (byteSize/1024.0) + # smaller than one GB + elif byteSize < 1073741824: return "%.1fM" % (byteSize/1048576.0) + # more than a GB + else: return "%.1fG" % (byteSize/1073741824.0) + + ''' + Returns the disk size of this file + ''' + def getDiskSize(self, path): + + return os.stat(path).st_size + + ''' + Returns the disk size of these DVO regions, i.e. the size on disk of all cpm and cpt files + ''' + def getDiskSizeOfDvoRegions(self, regions): + + size = 0.0 + for region in regions: + + # get combined size of cpm and cpt files + size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + ".cpm") + size = size + self.getDiskSize(self.config.dvoLocation + "/" + region + ".cpt") + + return size + + ''' + Synchronises MySQL database with the sky area currently set from DVO + Populates dvoDetections and dvoDone with FITS tables from DVO for the defined sky area, this + includes purging detections outside sky area + ''' + def sync(self): + + # go no further if we've already partly ingested a different DVO + if not self.correctDvo: return False + + # check we have something to do + if len(self.regionsToPurge) < 1 and len(self.regionsToIngest) < 1: + self.logger.infoPair("Nothing to ingest or purge", "finished ingest") + return True + + # make sure we have an up-to-date Images table + self.loadImages() + + # purge stuff + self.purgeRegions() + + totalRegions = len(self.regionsToIngest) + if totalRegions > 0: self.logger.infoSeparator() + + # now loop through all regions and ingest to MySQL + regionCount = 0 + for region in self.regionsToIngest: + + cpmPath = self.config.dvoLocation + "/" + region + ".cpm" + cptPath = self.config.dvoLocation + "/" + region + ".cpt" + + cpmTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpm") + cptTableName = self.scratchDb.getDbFriendlyTableName(region + ".cpt") + + self.logger.infoTitle("Region %d of %d: %s" % (regionCount+1, totalRegions, region)) + + # if we have not already ingested the cpm table, then do it now + if not self.scratchDb.alreadyImportedThisDvoTable(cpmPath): + + # import cpm table and index + self.importFits( + cpmPath, + "IMAGE_ID DET_ID OBJ_ID CAT_ID EXT_ID DB_FLAGS", + cpmTableName) + + self.scratchDb.createIndex(cpmTableName, "IMAGE_ID") + self.scratchDb.createIndex(cpmTableName, "CAT_ID") + self.scratchDb.createIndex(cpmTableName, "OBJ_ID") + + # shove SOURCE_IDs into measurement table + #self.logger.infoPair("Adding", "SOURCE_IDs") + #sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (SOURCE_ID SMALLINT)" + #self.scratchDb.execute(sql) + #sql = "UPDATE " + cpmTableName + " AS a, " + self.scratchDb.dvoImagesTable + " AS b \ + # SET a.SOURCE_ID = b.SOURCE_ID \ + # WHERE a.IMAGE_ID = b.IMAGE_ID" + #self.scratchDb.execute(sql) + + # we can now report that we have imported this table + self.scratchDb.setImportedThisDvoTable(cpmPath, region) + + # get the id for this cpm file so we can insert it in the dvoDetection table + fileId = self.scratchDb.getDvoFileId(cpmPath) + + # import cpt table and index + self.importFits( + cptPath, + "OBJ_ID CAT_ID EXT_ID", + cptTableName) + + self.scratchDb.createIndex(cptTableName, "IMAGE_ID") + self.scratchDb.createIndex(cptTableName, "CAT_ID") + self.scratchDb.createIndex(cptTableName, "OBJ_ID") + + self.logger.infoPair("Adding","PSPS objIDs") + # first try to add a column for PSPS_OBJ_ID. catch failure and continue + try: + sql = "ALTER TABLE " + cpmTableName + " ADD COLUMN (PSPS_OBJ_ID BIGINT)" + self.scratchDb.execute(sql) + except: + self.logger.infoPair("Already added PSPS_OBJ_ID column to", cpmTableName) + + # shove PSPS objIDs from cpt table into measurement table + sql = "UPDATE " + cpmTableName + " AS a, " + cptTableName + " AS b \ + SET a.PSPS_OBJ_ID = b.EXT_ID \ + WHERE a.CAT_ID = b.CAT_ID \ + AND a.OBJ_ID = b.OBJ_ID" + self.scratchDb.execute(sql) + + + # NB we use an INSERT IGNORE here. This is because of a known issue where multiple DVO cpm + # files can include the same detection, with the same IMAGE_ID/DET_ID pairing, but different + # PSPS object IDs assigned in the corresponding cpt file. This is believed to be a chip-boundary + # issue within DVO. So, for now, we take the first IMAGE_ID/DET_ID detection we find, and ignore the rest + self.logger.infoPair("Merging into", self.scratchDb.dvoDetectionTable) + sql = "INSERT IGNORE INTO " + self.scratchDb.dvoDetectionTable + " (\ + imageID \ + ,ippDetectID \ + ,detectID \ + ,ippObjID \ + ,objID \ + ,flags \ + ,fileID \ + ) SELECT \ + IMAGE_ID \ + ,DET_ID \ + ,EXT_ID \ + ,CAT_ID * 1000000000 + OBJ_ID \ + ,PSPS_OBJ_ID \ + ,DB_FLAGS \ + , " + str(fileId) + " \ + FROM " + cpmTableName + try: + self.scratchDb.execute(sql) + except: + self.logger.error("FAILED: " + sql) + continue + + # now drop what we don't need + self.logger.infoPair("Dropping tables", "%s and %s" % (cpmTableName, cptTableName)) + self.scratchDb.dropTable(cpmTableName) + self.scratchDb.dropTable(cptTableName) + + self.scratchDb.setImportedThisDvoTable(cptPath, region) + regionCount = regionCount + 1 + + self.logger.infoSeparator() + + self.logger.infoPair("Ingested", "%d DVO regions" % regionCount) + + # update lists after attempted sync + self.setSkyArea(self.minRa, self.maxRa, self.minDec, self.maxDec) + self.printSummary() + + return True + + ''' + Imports a FITS file. A regex filter lets you choose which tables to pull from the file + An optional final argument lets you name the resulatant database table + ''' + def importFits(self, path, columns, tableName): + + self.logger.infoPair("Importing file", "%s (%s)" % (path, self.getFileSizeStr(self.getDiskSize(path)))) + self.logger.debugPair("Writing to database table", tableName) + + tables = stilts.treads(path) + + count = 0 + for table in tables: + + self.logger.debugPair("Reading IPP table", table.name) + table = stilts.tpipe(table, cmd='explodeall') + + # IPP FITS files are littered with infinities, so remove these + self.logger.debugPair("Removing", "infinity values") + table = stilts.tpipe(table, cmd='replaceval -Infinity null *') + table = stilts.tpipe(table, cmd='replaceval Infinity null *') + + attempts = 0 + self.logger.debugPair("Writing table to database", table.name) + while True: + + try: + if attempts > 0: self.logger.infoPair("Attempt %d to write" % attempts, table.name) + table.cmd_keepcols(columns).write(self.scratchDb.url + '#' + tableName) + break + except: + # wait a few secs then try again + time.sleep(2) + attempts = attempts + 1 + + count = count + 1 + + self.logger.debugPair("Finished importing", tableName) + + return tableName + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/dxlayer.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/dxlayer.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/dxlayer.py (revision 33415) @@ -0,0 +1,74 @@ +#!/usr/bin/env jython + +import re +import sys +import tempfile +from xml.etree.ElementTree import ElementTree, Element, tostring +from subprocess import call, PIPE, Popen + +from batch import Batch + +''' +Class for using DXLayer webservice +''' +class DXLayer(object): + + ''' + Constructor + ''' + def __init__(self, logger): + + self.logger = logger + + ''' + Delete's a batch from the DXLayer + ''' + def deleteBatch(self, batchID): + + batch = Batch.getNameFromID(batchID) + + tempFile = tempfile.NamedTemporaryFile(mode='w+b') + + p = Popen("../perl/deleteFromDXLayer.pl -b " + batch + " -f " + tempFile.name, + shell=True, + stdout=PIPE) + p.wait() + + if p.returncode != 0: + self.logger.debugPair("DXLayer webservice", "failed") + return 0 + else: + self.logger.debugPair("DXLayer webservice", "successful") + + ret = self.decodeDXLayerResponse(tempFile.name) + + tempFile.close() + p.stdout.close() + + if ret: self.logger.infoPair("Deleted batch from DXLayer", batch) + else: self.logger.errorPair("Unable to delete batch from DXLayer", batch) + + return ret + + ''' + Parses the XML returned from the DXLayer webservice call + ''' + def decodeDXLayerResponse(self, dxLayerResponseFile): + + doc = ElementTree(file=dxLayerResponseFile) + + ret = False + try: + MESSAGE = doc.find("Rows/Row/Value").text + + #print "MESSAGE " + MESSAGE + if re.search("Path not found", MESSAGE): + ret = True + elif re.search("Deleted", MESSAGE): + ret = True + + except: + self.logger.error("Cannot read DXLayer response file") + ret = False + + return ret Index: branches/meh_branches/ppstack_test/ippToPsps/jython/fits.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/fits.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/fits.py (revision 33415) @@ -15,11 +15,11 @@ ''' - def __init__(self, logger, doc, originalPath): + def __init__(self, logger, config, originalPath): # set class variables self.originalPath = originalPath self.logger = logger - self.doc = doc - self.localDir = self.doc.find("localOutPath").text + self.config = config + self.header = None # does this file even exist? @@ -30,11 +30,12 @@ # ok, we have a file, now copy it locally to save on NFS overhead self.logger.debugPair("FITS file", self.originalPath) - self.localCopyPath = self.localDir + "/temp.fits" - shutil.copy2(self.originalPath, self.localCopyPath) + #self.localCopyPath = self.config.basePath + "/temp.fits" + #shutil.copy2(self.originalPath, self.localCopyPath) # open the local copy and parse the ridiculou plain-text header - self.file = open(self.localCopyPath) + #self.file = open(self.localCopyPath) + self.file = open(self.originalPath) self.header = self.parseHeader() - self.logger.debugPair("FITS local copy", self.localCopyPath) + #self.logger.debugPair("FITS local copy", self.localCopyPath) self.logger.debugPair("FITS primary header", "%d cards found" % len(self.header)) @@ -52,5 +53,6 @@ ''' def getPath(self): - return self.localCopyPath + #return self.localCopyPath + return self.originalPath ''' Index: branches/meh_branches/ppstack_test/ippToPsps/jython/gpc1db.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/gpc1db.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/gpc1db.py (revision 33415) @@ -20,6 +20,6 @@ Constructor ''' - def __init__(self, logger, doc): - super(Gpc1Db, self).__init__(logger, doc, "gpc1database") + def __init__(self, logger, config): + super(Gpc1Db, self).__init__(logger, config, "gpc1database") ''' @@ -32,35 +32,21 @@ ''' - TODO - ''' - def getIDsInThisDVODbForThisStageFudge(self): - - sql = "SELECT staticskyRun.sky_id \ - FROM staticskyInput, staticskyRun, stackRun, staticskyResult \ - WHERE staticskyRun.sky_id = staticskyInput.sky_id \ - AND staticskyInput.stack_id = stackRun.stack_id \ - AND staticskyInput.sky_id = staticskyResult.sky_id \ - AND staticskyRun.label like 'MD04.staticsky' \ - AND stackRun.filter like 'i%'" - - try: - rs = self.executeQuery(sql) - except: - self.logger.exception("Can't query for ids in DVO") + Gets a list of ids in this DVO database for this stage, could be cam or staticsky, for a box with defined center and side length + ''' + def getIDsInThisDVODbForThisStageInThisBox(self, dvoDb, batchType, ra=180., dec=0., side=181.): + + halfSide = side/2.0 + return self.getIDsInThisDVODbForThisStage(dvoDb, batchType, ra-halfSide, ra+halfSide, dec-halfSide, dec+halfSide) + + + ''' + Gets a list of ids in this DVO database for this stage, could be cam or staticsky, and a box can be defined with equatorial coordinates + ''' + def getIDsInThisDVODbForThisStage(self, dvoDb, batchType, minRA=-999, maxRA=999, minDec=-999, maxDec=999): + + self.logger.debugPair("GPC1 RA range", "%.2f -> %.2f" % (minRA, maxRA)) + self.logger.debugPair("GPC1 Dec range", "%.2f -> %.2f" % (minDec, maxDec)) ids = [] - while (rs.next()): - ids.append(rs.getInt(1)) - - rs.close() - - self.logger.debug("Found %d items in DVO database '" % (len(ids))) - - return ids - - ''' - Gets a list of ids in this DVO database for this stage, could be cam or staticsky (so far) - ''' - def getIDsInThisDVODbForThisStage(self, dvoDb, batchType): if batchType == "P2": # TODO define these someplace @@ -71,15 +57,21 @@ JOIN minidvodbRun USING(minidvodb_name) \ JOIN minidvodbProcessed USING(minidvodb_id) \ - WHERE minidvodbRun.minidvodb_group = '" + dvoDb + "' \ + JOIN camRun ON (stage_id = cam_id) \ + JOIN chipRun USING (chip_id) \ + JOIN rawExp USING (exp_id) \ + JOIN mergedvodbRun using (minidvodb_id) \ + WHERE mergedvodbRun.mergedvodb = '" + dvoDb + "' \ AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ AND minidvodbProcessed.fault = 0 \ AND addRun.stage = '" + stage + "' \ - AND addRun.state = 'full'" + AND addRun.state = 'full' \ + AND decl BETWEEN RADIANS(" + str(minDec) + ") AND RADIANS(" + str(maxDec) + ") \ + AND ra BETWEEN RADIANS(" + str(minRA) + ") AND RADIANS(" + str(maxRA) + ")" elif batchType == "ST": - stage = "staticsky" - sql = "SELECT DISTINCT stack_id \ - FROM staticskyInput \ + stage = "staticsky_multi" + sql = "SELECT DISTINCT stack_id FROM staticskyInput \ JOIN addRun ON(staticskyInput.sky_id = addRun.stage_id) \ JOIN minidvodbRun USING(minidvodb_name) \ @@ -90,15 +82,13 @@ AND addRun.stage = '" + stage + "' \ AND addRun.state = 'full'" - - try: - rs = self.executeQuery(sql) + + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() except: self.logger.exception("Can't query for ids in DVO") - - ids = [] - while (rs.next()): - ids.append(rs.getInt(1)) - - rs.close() + return ids + self.logger.debug("Found %d items in DVO database '%s' for stage='%s'" % (len(ids), dvoDb, stage)) @@ -154,10 +144,14 @@ rs = self.executeQuery(sql) rs.first() + except: + self.logger.errorPair("Can't query for", "stack meta data") + + try: meta.append(rs.getString(1)) meta.append(rs.getString(2)) meta.append(rs.getString(3)) except: - self.logger.exception("Can't query for stack meta") - + self.logger.errorPair("getStackStageMeta()", "empty meta data") + return meta @@ -178,4 +172,8 @@ rs = self.executeQuery(sql) rs.first() + except: + self.logger.errorPair("Can't query for", "camera meta data") + + try: meta.append(rs.getInt(1)) meta.append(rs.getString(2)) @@ -185,5 +183,5 @@ meta.append(rs.getString(6)) except: - self.logger.exception("Can't query for camera meta") + self.logger.errorPair("getCameraStageMeta()", "empty meta data") return meta @@ -205,9 +203,13 @@ rs.first() except: - self.logger.exception("Can't query for camera smfs") - - # get path to base dir of cmf files - path = rs.getString(1) - path = path[0:path.rfind("/")] + self.logger.errorPair("Can't query for smf file for cam_id", str(camID)) + return None + + try: + # get path to base dir of cmf files + path = rs.getString(1) + except: + self.logger.errorPair("No smf files found for cam_id", str(camID) ) + return None # list all smf files if a neb path @@ -215,5 +217,5 @@ if path.startswith("neb"): - f=os.popen("neb-ls -p "+path+"/%smf") + f=os.popen("neb-ls -p "+path+"%smf") for i in f.readlines(): files.append(i.rstrip()) @@ -221,24 +223,64 @@ # or not a neb path else: - files = glob.glob(path + "/*.smf") + files = glob.glob(path + ".smf") if len(files) < 1: return None - return Fits(self.logger, self.doc, files[0]) # TODO just returning first file - check + return Fits(self.logger, self.config, files[0]) # TODO just returning first file - check ''' Gets the path to the cmf file for this stack_id - ''' - def getStackStageCmf(self, stackID): - - sql = "SELECT staticskyResult.path_base \ - FROM staticskyRun \ - JOIN staticskyInput USING(sky_id) \ + + The logic here is fairly ridiculous due to issues with how GPC1 stores info regarding multi-filter stacks. In partcular, it + is impossible to find a full path for a cmf file for a given stack_id + + Instead, we get a path_base to a directory with a collection of cmfs. neb-ls will list 5 files, but a number < 5 may actually exist + meaning that neb-ls -p will fail. + + our logic is as follows: + + - use stack_id and the DVO label to find a path base for all cmfs + - loop through all the potential cmfs, run neb-ls -p on each + - if neb-ls -p fails, then the file does not exists and we ignore it + - if neb-ls -p succeeds, we open that cmf file and check the STK_ID value in its header to see if it is the one we want + - if it is, great, exit + - if not, try the next one in the list + + ''' + def getStackStageCmf(self, dvoDb, stackID): + + ''' + sql = "SELECT DISTINCT staticskyResult.path_base \ + FROM staticskyRun \ + JOIN staticskyInput USING(sky_id) \ JOIN staticskyResult USING(sky_id) \ - JOIN stackRun USING(stack_id) \ + JOIN stackRun USING(stack_id) \ JOIN stackSumSkyfile USING(stack_id) \ - WHERE stack_id = %s" % stackID - + JOIN addRun ON sky_id = stage_id \ + JOIN minidvodbRun USING (minidvodb_name) \ + WHERE stack_id = " + str(stackID) + " \ + AND minidvodbRun.minidvodb_group = '" + dvoDb + "' \ + AND minidvodbRun.state = 'merged' \ + AND stage = 'staticsky_multi'" + ''' + + sql = "SELECT DISTINCT staticskyResult.path_base \ + FROM staticskyRun \ + JOIN staticskyInput USING(sky_id) \ + JOIN staticskyResult USING(sky_id) \ + JOIN stackRun USING(stack_id) \ + JOIN stackSumSkyfile USING(stack_id) \ + JOIN addRun ON sky_id = stage_id \ + JOIN minidvodbRun USING (minidvodb_name) \ + JOIN mergedvodbRun using (minidvodb_id) \ + WHERE stack_id = " + str(stackID) + " \ + AND minidvodbRun.state = 'merged' \ + AND mergedvodbRun.state = 'full' \ + AND mergedvodbRun.mergedvodb = '" + dvoDb + "' \ + AND stage = 'staticsky_multi'" + + + # get single path base for the directory containing all cmf files, one of which is hopefully for our stack_id try: rs = self.executeQuery(sql) @@ -246,19 +288,38 @@ pathBase = rs.getString(1) - # now find ALL cmf files at this path_base as there can be more than one - f=os.popen("neb-ls -p " + pathBase + "%cmf") - for i in f.readlines(): - - path = i.rstrip() - fits = Fits(self.logger, self.doc, path) - - # we need to check if this is the correct cmf file, and if so then break and return - if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): - return fits - - except: - self.logger.exception("Can't query for stack cmfs") - - self.logger.error("Could not find stack cmf") + # now find and loop through all cmf files at this path_base + files=os.popen("neb-ls " + pathBase + "%cmf") + for i in files.readlines(): + + nebPath = i.rstrip() + + # now attempt to run 'neb-ls -p' to actually read this cmf file + try: + self.logger.debug("Trying to neb-ls -p on '" + nebPath + "'") + paths=os.popen("neb-ls -p " + nebPath) + path = paths.readline().rstrip() + if len(path) < 1: + self.logger.debug("zero length path - skipping") + raise + + # if we get here, then the cmf is readable, now check the stack_id + fits = Fits(self.logger, self.config, path) + if fits.getPrimaryHeaderValue("STK_ID") == str(stackID): + # we have the right file! + self.logger.debug("SUCCESS!: %s == %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) + return fits + else: + # this is not the correct file + self.logger.debug("STACK ID: %s != %s"% (fits.getPrimaryHeaderValue("STK_ID"), str(stackID))) + + # an exception here means that nebulous could not read this file, so we just skip it + except: + self.logger.debug("NEB FAILED SKIPPING") + pass + + except: + self.logger.errorPair("Can't query for stack cmfs", "stack_id = %d" % stackID) + + self.logger.errorPair("Couldn't find stack cmf for", "stack_id = %d" % stackID) return None Index: branches/meh_branches/ppstack_test/ippToPsps/jython/initbatch.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/initbatch.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/initbatch.py (revision 33415) @@ -27,16 +27,19 @@ def __init__(self, logger, - configPath, - configDoc, + config, gpc1Db, - ippToPspsDb): + ippToPspsDb, + scratchDb, + batchID): super(InitBatch, self).__init__(logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, 0, + batchID, "IN", - None) + None, + 0) self.outputFitsFile = "00000000.FITS"; @@ -49,4 +52,9 @@ self.logger.debug("importIppTables method here to satidfy base-class") + ''' + Here to satidfy the base-class + ''' + def alterPspsTables(self): + return True ''' Index: branches/meh_branches/ppstack_test/ippToPsps/jython/ipptopsps.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/ipptopsps.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/ipptopsps.py (revision 33415) @@ -1,171 +1,152 @@ #!/usr/bin/env jython -# The starting point for ippTopsps. See usage below. +import signal +import sys +import os +import time +import logging +import socket -import time -import sys -import logging.config -from xml.etree.ElementTree import ElementTree, Element, tostring - +from config import Config from pslogger import PSLogger -from gpc1db import Gpc1Db from ipptopspsdb import IppToPspsDb - -from initbatch import InitBatch -from stackbatch import StackBatch -from detectionbatch import DetectionBatch - -''' -Queues up all available stuff for this batch type -''' -def queueItems(batchType): - - logger.infoTitle("Queuing " + batchType + " items") - - allIDs = [] - - # see if there are any IDs listed in the config - try: - for element in configDoc.findall('options/ids/id'): - allIDs.append(int(element.text)) - except: - logger.errorPair("Problem loading IDs from config under", "options/ids/id") - - # no config dictated IDs, so get from DVO instead - if len(allIDs) > 0: - logger.infoPair("Using config IDs (not DVO)", "%d" % len(allIDs)) - else: - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOGPC1LABEL, batchType) - logger.infoPair("All %s items in DVO" % batchType, "%d" % len(allIDs)) - - # if in FORCE mode, then queue full list - if FORCE: ids = allIDs - - # if not in TEST mode, then only queue un-processed items - else: - processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOGPC1LABEL) - logger.infoPair("Processed %s items" % batchType, "%d" % len(processedIDs)) - ids = list(set(allIDs) - set(processedIDs)) - logger.infoPair("Unprocessed %s items" % batchType, "%d" % len(ids)) - - logger.infoPair("%s items queued" % batchType, "%d" % len(ids)) - - # loop round IDs of all items to be processed - for id in ids: - - if not FORCE and ippToPspsDb.alreadyProcessed(id): continue - - if batchType == "P2": - batch = DetectionBatch(logger, - CONFIG, - configDoc, - gpc1Db, - ippToPspsDb, - id) - elif batchType == "ST": - batch = StackBatch(logger, - CONFIG, - configDoc, - gpc1Db, - ippToPspsDb, - id) - - batch.run() - - # if in TEST mode, then quit after one batch - if TEST: break +from dxlayer import DXLayer +from datastore import Datastore +from dvo import Dvo +from odm import Odm ''' -Start of program. +Abstract base class for all ippToPsps programs +handle command-line args, setting up of Config and logger objects, clean exiting etc ''' +class IppToPsps(object): -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " [init]" - sys.exit(1) + ''' + Constructor + ''' + def __init__(self, argv): -if len(sys.argv) > 2 and sys.argv[2] == "init": - QUEUE_IN = 1 -else: - QUEUE_IN = 0 + # are the arsg ok? all programs require a config name + if len(sys.argv) < 2: + self.printUsage() + self.exitProgram("Wrong args") + + # some class constants + self.PROGNAME = sys.argv[0] + self.CONFIGNAME = sys.argv[1] + self.HOST = socket.gethostname() + self.PID = os.getpid() + self.PAUSEPERIOD = 60 + self.HOURS = None + self.MINUTES = None + self.SECONDS = None -# open config file -configDoc = ElementTree(file=CONFIG) -TEST = int(configDoc.find("options/testMode").text) + # set up config object + self.config = Config(self.PROGNAME, self.CONFIGNAME) + self.logger = self.config.getLogger(self.HOST, self.PID, 1, 1) -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("ippToPspsLog") + # create connection to databases database + self.ippToPspsDb = IppToPspsDb(self.logger, self.config) -# if in test mode, print log to screen, otherwise, only to file -logger.setup(configDoc, "ippToPsps", TEST, not TEST) + self.checkClientStatus() -# create database objects -gpc1Db = Gpc1Db(logger, configDoc) -ippToPspsDb = IppToPspsDb(logger, configDoc) + # catch Ctrl-C signal + signal.signal(signal.SIGINT, self.signal_handler) -# check we connected ok -if not gpc1Db.everythingOK: sys.exit(1) -if not ippToPspsDb.everythingOK: sys.exit(1) + # title for log + self.logger.infoSeparator() + self.logger.infoTitle("ippToPsps '" + self.PROGNAME + "' started") -# get values from the configutaion file -POLLPERIOD = 600 -DVOGPC1LABEL = configDoc.find("dvo/gpc1Label").text -FORCE = int(configDoc.find("options/force").text) -EPOCH = configDoc.find("options/epoch").text -PUBLISH = int(configDoc.find("options/publishToDatastore").text) -QUEUE_P2 = int(configDoc.find("options/queueP2").text) -QUEUE_ST = int(configDoc.find("options/queueST").text) + ''' + Parses a poll-time argument string and stored as class variables + ''' + def parsePollTimeArg(self, hours): -# prompt user: FORCE and PUBLISH is a dangerous combination -if FORCE and PUBLISH and not QUEUE_IN: - response = raw_input("\n*** Are you sure you want to publish data with the 'force' option enabled (y/n)? ") - if response != "y": sys.exit(1) - -# prompt user: TEST and PUBLISH is a dangerous combination -if TEST and PUBLISH and not QUEUE_IN: - response = raw_input("\n*** Are you sure you want to publish data with the 'testMode' option enabled (y/n)? ") - if response != "y": sys.exit(1) - -# print info to log -logger.infoSeparator() -logger.infoTitle("ippToPsps started") -logger.infoPair("Configuration file", CONFIG) -logger.infoPair("Loading epoch", EPOCH) -logger.infoPair("DVO gpc1 label", DVOGPC1LABEL) -logger.infoBool("Queuing P2 batches?", QUEUE_P2) -logger.infoBool("Queuing ST batches?", QUEUE_ST) -logger.infoBool("Forcing?", "%d" % FORCE) -logger.infoBool("Test mode?", "%d" % TEST) -logger.infoBool("Publishing?", PUBLISH) - -''' -Main processing loop: - - 1) creates an IN batch if requested - 2) queues available P2 batches (if requested) - 3) queues available ST batches (if requested) - 4) if in test mode, quits - 5) otherwise, waits then repeats from 2 - -''' -while True: - - # queue all items for all batch types, sequentially - if QUEUE_IN: - batch = InitBatch(logger, CONFIG, configDoc, gpc1Db, ippToPspsDb) - batch.run() - break; - - if QUEUE_P2: queueItems("P2") - if QUEUE_ST: queueItems("ST") - - # in the TEST mode, we quit after submitting one batch - if TEST: break - - # wait for the POLLPERIOD before checking for new ids - logger.infoPair("Finished. Waiting for", "%.1f minutes before checking DVO for new items" % (POLLPERIOD/60.0)) - time.sleep(POLLPERIOD) + try: + self.HOURS = float(hours) + self.MINUTES = self.HOURS * 60.0 + self.SECONDS = self.MINUTES * 60.0 + except: + pass + ''' + Waits for user defined poll time + ''' + def waitForPollTime(self): + + # if not POLL time set, then return False + if not self.SECONDS: return False + + if self.HOURS < 1.0: + str = "%.1f minutes" % self.MINUTES + else: + str = "%.1f hours" % self.HOURS + + self.logger.infoPair("Waiting for", str) + time.sleep(self.SECONDS) + + return True + + ''' + Prints usage to stdout + ''' + def printUsage(self): + # using sys.stdout to avoid newline in case sub-classes need to add further args + sys.stdout.write("** Usage: " + sys.argv[0] + " ") + + + ''' + Refreshes the config then recreates objects with new settings + ''' + def refreshConfig(self): + return self.ippToPspsDb.readConfig(self.HOST, self.PID) + + ''' + All implementing subclasses (programs) should call this method once in a while to see if it should be paused or killed, otherwise this updates the clients table with current time and reloads the config data + ''' + def checkClientStatus(self): + + self.ippToPspsDb.updateClient(self.PROGNAME, self.HOST, self.PID, self.config.name) + + if self.ippToPspsDb.isKilled(self.HOST, self.PID): + self.exitProgram("killed via Db") + + # this loop pauses the process if we have no config or we have set it to pause + while not self.refreshConfig() or self.ippToPspsDb.isPaused(self.HOST, self.PID): + + self.ippToPspsDb.updateClient(self.PROGNAME, self.HOST, self.PID, self.config.name) + if self.ippToPspsDb.isKilled(self.HOST, self.PID): self.exitProgram("killed while paused") + self.logger.infoPair("Paused. Will check again in", "%d secs" % self.PAUSEPERIOD) + time.sleep(self.PAUSEPERIOD) + + + ''' + Handler for Ctrl-C signal + ''' + def signal_handler(self, signal, frame): + + self.exitProgram("Ctrl-C") + + + ''' + Exits the program cleanly, removes info from ipptopsps database + ''' + def exitProgram(self, exitReason): + + # write message to log or stdout if no logger set up + try: + self.logger.infoPair("Program exited", exitReason) + except: + print "** program exited: " + exitReason + + # remove from database if possible + try: + self.ippToPspsDb.removeClient(self.HOST, self.PID) + except: + pass + + sys.exit(0) + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/ipptopspsdb.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/ipptopspsdb.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/ipptopspsdb.py (revision 33415) @@ -17,54 +17,142 @@ Constructor ''' - def __init__(self, logger, doc): - super(IppToPspsDb, self).__init__(logger, doc, "ipptopspsdatabase") - - ''' - Creates a new batch - ''' - def createNewBatch(self, batchType, stageID, survey, dvoDb, datastoreProduct): - - sql = "INSERT INTO batch ( \ - batch_type, \ - stage_id, \ - survey, \ - dvo_db, \ - datastore_product \ - ) VALUES ( \ - '" + batchType + "', \ - " + str(stageID) + ", \ - '" + survey + "', \ - '" + dvoDb + "', \ - '" + datastoreProduct + "' \ - )" - - self.execute(sql) - - sql = "SELECT MAX(batch_id) FROM batch" - - batchID = -1; - - try: - rs = self.executeQuery(sql) - rs.first() - batchID = rs.getInt(1) - except: - self.logger.exception("Unable to get batch ID") - - self.logger.debug("Created new batch in ippToPsps database with batchID = %d" % batchID) - - return batchID; - - ''' - Returns a list of processed batch IDs that are merged but not yet deleted - ''' - def getMergedButNotDeletedBatchIDs(self, epoch, dvoGpc1Label): + def __init__(self, logger, config): + super(IppToPspsDb, self).__init__(logger, config, "ipptopspsdatabase") + + self.MAX_FAILS = 5 + + ''' + Returns a list of merged batch IDs that are merged but not yet deleted + ''' + def getMergedButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column): sql = "SELECT DISTINCT batch_id \ FROM batch \ WHERE timestamp > '" + epoch + "' \ - AND dvo_db = '" + dvoGpc1Label + "' \ - AND merged \ - AND !deleted" + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND merged = 1 \ + AND " + column + " = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ''' + Returns a list of batch IDs marked as 'purged' but not yet deleted + ''' + def getPurgedButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND purged = 1 \ + AND " + column + " = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that have been loaded to the ODM but not yet deleted + ''' + def getLoadedToODMButNotDeletedBatchIDs(self, batchType, epoch, dvoGpc1Label, column): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND (loaded_to_ODM = -1 OR merge_worthy = 1) \ + AND " + column + " = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + + self.logger.debug("Found %d merged but un-deleted items" % len(ids)) + + return ids + + ''' + Returns a list of merged batch IDs that are not deleted from local disk + ''' + def getMergedButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label): + return self.getMergedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local") + + ''' + Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from local disk + ''' + def getLoadedToODMButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label): + return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local") + + ''' + Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from datastore + ''' + def getLoadedToODMButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label): + return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_datastore") + + ''' + Returns a list of processed batch IDs that are loaded to the ODM, but not deleted from DXLayer + ''' + def getLoadedToODMButNotDeletedFromDXLayer(self, batchType, epoch, dvoGpc1Label): + return self.getLoadedToODMButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_dxlayer") + + ''' + Returns a list of purged batch IDs that not deleted from local disk + ''' + def getPurgedButNotDeletedFromLocalDisk(self, batchType, epoch, dvoGpc1Label): + return self.getPurgedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_local") + + ''' + Returns a list of purged batch IDs that are not deleted from datastore + ''' + def getPurgedButNotDeletedFromDatastore(self, batchType, epoch, dvoGpc1Label): + return self.getPurgedButNotDeletedBatchIDs(batchType, epoch, dvoGpc1Label, "deleted_datastore") + + + ''' + Returns a list of processed batch IDs that are not loaded to the datastore + NB we have a date constraint here because, before this time, it is possible that + a test batch would be labelled as processed but not loaded to datastore, and we would NOT + want to load a test batch to PSPS + + We also check that these batches are more than 4 HOURs old, so that we don't attempt to + publish something simulataneously with the client that it producing it + ''' + def getProcessedButFailedDatastoreBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND processed = 1\ + AND loaded_to_datastore != 1 \ + AND timestamp > '2011-10-27' \ + AND timestamp < now() - INTERVAL 4 HOUR" ids = [] @@ -74,23 +162,26 @@ ids.append(rs.getInt(1)) except: - self.logger.exception("Can't query for merged batch ids in ipptopsps Db") + self.logger.exception("Can't query for processed batch ids not loaded to datastore ipptopsps Db") rs.close() - self.logger.debug("Found %d merged but un-deleted items" % len(ids)) - - return ids - - ''' - Returns a list of processed batch IDs that are not yet merged for this epoch and dvo label - ''' - def getProcessedButUnmergedBatchIDs(self, epoch, dvoGpc1Label): - - sql = "SELECT DISTINCT batch_id \ + self.logger.debug("Found %d processed but-not-on-datastore items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that are not yet loaded to the ODM + NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch + ''' + def getUnloadedBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ FROM batch \ WHERE timestamp > '" + epoch + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND loaded_to_datastore \ - AND !merged" + AND loaded_to_datastore = 1 \ + AND batch_type = '" + batchType + "' \ + AND loaded_to_ODM = 0 \ + AND deleted_datastore = 0" ids = [] @@ -100,24 +191,133 @@ ids.append(rs.getInt(1)) except: - self.logger.exception("Can't query for processed batch ids in ipptopsps Db") + self.logger.exception("Can't query for unloaded batch ids in ipptopsps Db") rs.close() - self.logger.debug("Found %d processed and un-merged items" % len(ids)) - - return ids - - ''' - Returns a list of IDs for this batch type, epoch and dvo label (i.e. either - cam_ids or stack_ids) for which the spefified column is true (=1) - ''' - def getStageIDs(self, batchType, epoch, dvoGpc1Label, column): - - sql = "SELECT DISTINCT stage_id \ + self.logger.debug("Found %d unloaded items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that are not yet merge_worthy and that have not failed to load + NOTE we exclude batches already deleted from the datastore, as sometimes we delete stuff prior to loading to ODM if there is a known problem with the batch + ''' + def getUnmergeWorthyBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND loaded_to_datastore = 1 \ + AND batch_type = '" + batchType + "' \ + AND merge_worthy = 0 \ + AND loaded_to_ODM != -1 \ + AND deleted_datastore = 0" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for un-mergeworthy batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d unloaded items" % len(ids)) + + return ids + + ''' + Returns a list of processed batch IDs that are merge_worthy, but not yet merged, for this epoch and dvo label + ''' + def getUnmergedBatchIDs(self, epoch, dvoGpc1Label, batchType): + + sql = "SELECT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND merge_worthy = 1 \ + AND batch_type = '" + batchType + "' \ + AND merged != 1" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for unmerged batch ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d un-merged items" % len(ids)) + + return ids + + ''' + Returns a list of batch IDs for this epoch and dvo label for which the specified column is set to value + ''' + def getBatchIDs(self, epoch, dvoGpc1Label, column, value): + + sql = "SELECT DISTINCT batch_id \ + FROM batch \ + WHERE timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND " + column + " = " + str(value) + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for batch ids with " + column + " = " + str(value) + " in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d batches with %s = %d" % (len(ids), column, value)) + + return ids + + ''' + Returns the total detections published to the datastore for this epoch, dvo label and batch type + ''' + def getTotalDetectionsPublished(self, batchType, epoch, dvoGpc1Label): + + sql = "SELECT SUM(total_detections) \ FROM batch \ WHERE batch_type = '" + batchType + "' \ AND timestamp > '" + epoch + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ - AND " + column + " = 1" + AND processed = 1 \ + AND loaded_to_datastore = 1" + + total = -1 + try: + rs = self.executeQuery(sql) + rs.first() + total = rs.getLong(1) + except: + self.logger.exception("Can't query for total detections published") + + rs.close() + + self.logger.debug("Found %d detections" % total) + + return total + + ''' + Returns a list of IDs for this batch type, epoch and dvo label (i.e. either + cam_ids or stack_ids) for which the specified column is set to value + ''' + def getStageIDs(self, batchType, epoch, dvoGpc1Label, column, value): + + sql = "SELECT DISTINCT stage_id \ + FROM batch \ + WHERE batch_type = '" + batchType + "' \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND " + column + " = " + str(value) ids = [] @@ -127,5 +327,5 @@ ids.append(rs.getInt(1)) except: - self.logger.exception("Can't query for ids with " + column + " = 1 in ipptopsps Db") + self.logger.exception("Can't query for ids with " + column + " = " + str(value) + " in ipptopsps Db") rs.close() @@ -136,16 +336,61 @@ ''' + Series of getter methods that utilise getBatchIDs() method above + ''' + def getloadedToDatastoreBatchIDs(self, epoch, dvoGpc1Label): + return self.getBatchIDs(epoch, dvoGpc1Label, "loaded_to_datastore", 1) + def getFailedLoadedToODMBatchIDs(self, epoch, dvoGpc1Label): + return self.getBatchIDs(epoch, dvoGpc1Label, "loaded_to_ODM", -1) + + ''' + TODO remove these Series of getter methods that utilise getStageIDs() method above ''' def getProcessedIDs(self, batchType, epoch, dvoGpc1Label): - return self.getStageIDs(batchType, epoch, dvoGpc1Label, "processed") - def getloadedToDatastoreIDs(self, batchType, epoch, dvoGpc1Label): - return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_datastore") + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "processed", 1) + def getFailedProcessedIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "processed", -1) + def getLoadedToDatastoreIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_datastore", 1) + def getFailedLoadedToDatastoreIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_datastore", -1) def getLoadedToODMIDs(self, batchType, epoch, dvoGpc1Label): - return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_ODM") - def getMergeWothyIDs(self, batchType, epoch, dvoGpc1Label): - return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merge_worthy") + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_ODM", 1) + def getFailedLoadedToODMIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "loaded_to_ODM", -1) + def getMergeWorthyIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merge_worthy", 1) + def getFailedMergeWorthyIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merge_worthy", -1) def getMergedIDs(self, batchType, epoch, dvoGpc1Label): - return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merged") + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merged", 1) + def getFailedMergedIDs(self, batchType, epoch, dvoGpc1Label): + return self.getStageIDs(batchType, epoch, dvoGpc1Label, "merged", -1) + + ''' + Returns a list of stages + ''' + def getStages(self): + return ['processed', 'loaded_to_datastore', 'loaded_to_ODM', 'merge_worthy', 'deleted_datastore', 'deleted_dxlayer', 'merged', 'deleted_local' ] + + ''' + Returns a count of batches processed in the specified time period. + ''' + def countBatchesInLastPeriod(self, batchType, epoch, dvoGpc1Label, interval): + + sql = "SELECT COUNT(batch_id) \ + FROM batch \ + WHERE batch_type = '" + batchType + "' \ + AND processed = 1 \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND timestamp BETWEEN now() - INTERVAL " + interval + " AND now()" + + try: + rs = self.executeQuery(sql) + if rs.first(): return rs.getInt(1) + except: + self.logger.exception("Unable to count batches in interval") + return 0 ''' @@ -155,10 +400,10 @@ def getTimeOfLastBatchPublished(self, batchType, epoch, dvoGpc1Label): - minutes = None - - sql = "SELECT TIMESTAMPDIFF(MINUTE, timestamp, now()) \ + seconds = None + + sql = "SELECT TIMESTAMPDIFF(SECOND, timestamp, now()) \ FROM batch \ WHERE batch_type = '" + batchType + "' \ - AND loaded_to_datastore \ + AND loaded_to_datastore = 1 \ AND timestamp > '" + epoch + "' \ AND dvo_db = '" + dvoGpc1Label + "' \ @@ -167,15 +412,17 @@ try: rs = self.executeQuery(sql) - if rs.first(): minutes = rs.getFloat(1) + if rs.first(): seconds = rs.getFloat(1) except: self.logger.exception("Unable to get last batch published") - if not minutes: return "Never" - + if not seconds: return "Never" + + minutes = seconds/60.0 hours = minutes/60.0 - days = float(hours)/24.0 - weeks = float(days)/7.0 - - if minutes < 60: return "%.1f minutes ago" % minutes + days = hours/24.0 + weeks = days/7.0 + + if seconds < 60: return "%.1f secs ago" % seconds + elif minutes < 60: return "%.1f mins ago" % minutes elif hours < 48: return "%.1f hours ago" % hours elif days < 7: return "%.1f days ago" % days @@ -183,63 +430,12 @@ ''' - TODO - ''' - def getBatchesPerDay(self, batchType, startTime, endTime=""): - - bph = self.getBatchesPerHour(batchType, startTime, endTime) - return bph * 24 - - ''' - TODO - ''' - def getBatchesPerHour(self, batchType, startTime, endTime=""): - - bpm = self.getBatchesPerMinute(batchType, startTime, endTime) - return bpm * 60 - - ''' - TODO - ''' - def getAverageTimePerBatch(self, batchType, startTime, endTime=""): - - sql = "SELECT (TIMESTAMPDIFF(MINUTE, min(timestamp), max(timestamp)) / COUNT(*)) \ - FROM batch \ - WHERE timestamp > '" + startTime + "' and batch_type = '" + batchType + "'" - - try: - rs = self.executeQuery(sql) - rs.first() - time = rs.getDouble(1) - except: - self.logger.exception("Unable to get batches per minute") - - return time - - ''' - TODO - ''' - def getBatchesPerMinute(self, batchType, startTime, endTime=""): - - sql = "SELECT (count(*) / TIMESTAMPDIFF(MINUTE, min(timestamp), max(timestamp)) ) \ - FROM batch \ - WHERE timestamp > '" + startTime + "' and batch_type = '" + batchType + "'" - - try: - rs = self.executeQuery(sql) - rs.first() - bpm = rs.getDouble(1) - except: - self.logger.exception("Unable to get batches per minute") - - return bpm - - ''' Updates min/max object ID on this table and batch ''' - def updateMinMaxObjID(self, batchID, minObjID, maxObjID): + def updateDetectionStats(self, batchID, minObjID, maxObjID, totalDetections): sql = "UPDATE batch SET \ min_obj_id = " + str(minObjID) + ", \ - max_obj_id = " + str(maxObjID) + " \ + max_obj_id = " + str(maxObjID) + ", \ + total_detections = " + str(totalDetections) + " \ WHERE batch_id = " + str(batchID) @@ -257,5 +453,4 @@ self.execute(sql) - ''' Updates batch processed field @@ -265,8 +460,20 @@ ''' - Updates batch deleted field - ''' - def updateDeleted(self, batchID, deleted): - self.updateColumn(batchID, "deleted", deleted) + Updates batch deleted local field + ''' + def updateDeletedLocal(self, batchID, value): + self.updateColumn(batchID, "deleted_local", value) + + ''' + Updates batch deleted datastore field + ''' + def updateDeletedDatastore(self, batchID, value): + self.updateColumn(batchID, "deleted_datastore", value) + + ''' + Updates batch deleted dxlayer field + ''' + def updateDeletedDxlayer(self, batchID, value): + self.updateColumn(batchID, "deleted_dxlayer", value) ''' @@ -290,13 +497,14 @@ ''' - Have we already processed and published this batch? - ''' - def alreadyProcessed(self, stage_id): + Is someone processing this item right now? Checks whether this was attempted within the last 2 HOUR + ''' + def processingNow(self, batchType, stage_id, epoch, dvoGpc1Label): sql = "SELECT COUNT(*) \ FROM batch \ WHERE stage_id = " + str(stage_id) + " \ - AND processed \ - AND loaded_to_datastore" + AND batch_type = '" + batchType + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND timestamp BETWEEN now() - INTERVAL 4 HOUR AND now()" try: @@ -304,11 +512,151 @@ rs.first() if rs.getInt(1) > 0: - self.logger.error("Batch with stage_id = "+str(stage_id) + " has already been processed and published to datastore") + self.logger.debugPair(str(stage_id) + " is already being processed", "skipping") return True else: return False except: + self.logger.exception("Unable to check whether this batch is being processed") + + ''' + Gets the batch type of this batch + ''' + def getBatchType(self, batch_id): + + sql = "SELECT batch_type \ + FROM batch \ + WHERE batch_id = " + str(batch_id) + + batch_type = "NA" + try: + rs = self.executeQuery(sql) + rs.first() + batch_type = rs.getString(1) + except: self.logger.exception("Unable to check whether this batch has already been processed") - + + return batch_type + + ''' + Have we already processed this stage_id? + ''' + def alreadyProcessed(self, batchType, stage_id, epoch, dvoGpc1Label): + + sql = "SELECT COUNT(*) \ + FROM batch \ + WHERE stage_id = " + str(stage_id) + " \ + AND batch_type = '" + batchType + "' \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND processed = 1" + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.debugPair(str(stage_id) + " has already been published", "skipping") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this batch has already been processed") + + ''' + Has this stage_id consistently failed to process? + ''' + def consistentlyFailed(self, batchType, stage_id, epoch, dvoGpc1Label): + + sql = "SELECT COUNT(*) \ + FROM batch \ + WHERE stage_id = " + str(stage_id) + " \ + AND batch_type = '" + batchType + "' \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND processed = -1" + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) >= self.MAX_FAILS: + self.logger.errorPair(str(stage_id) + " has failed %d times" % self.MAX_FAILS, "skipping") + return True + else: + return False + except: + self.logger.exception("Unable to check whether this batch has consistently failed") + + ''' + Returns a list of stage_ids that have failed the max number of times + ''' + def getConsistentlyFailedIDs(self, batchType, epoch, dvoGpc1Label): + + sql = "SELECT DISTINCT stage_id, COUNT(stage_id) AS numoccur \ + FROM batch \ + WHERE batch_type = '" + batchType + "' \ + AND timestamp > '" + epoch + "' \ + AND dvo_db = '" + dvoGpc1Label + "' \ + AND processed = -1 \ + GROUP BY stage_id HAVING numoccur >= " + str(self.MAX_FAILS) + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): + ids.append(rs.getInt(1)) + except: + self.logger.exception("Can't query for consistantly failed ids in ipptopsps Db") + + rs.close() + + self.logger.debug("Found %d consistantly failed stage_ids for batch type='%s'" % (len(ids), batchType)) + + return ids + + ''' + Locks the batch table. This will wait if the lock is held by someone else + ''' + def lockBatchTable(self): + self.lockTable("batch") + + ''' + Creates a new batch. This is done with a locked table in order that no two clients can start work on the same item + ''' + def createNewBatch(self, batchType, stageID, config): + + batchID = -1; + + if config.force or \ + (not self.alreadyProcessed(batchType, stageID, config.epoch, config.dvoLabel) \ + and not self.processingNow(batchType, stageID, config.epoch, config.dvoLabel) \ + and not self.consistentlyFailed(batchType, stageID, config.epoch, config.dvoLabel)): + + sql = "INSERT INTO batch ( \ + batch_type, \ + stage_id, \ + survey, \ + dvo_db, \ + datastore_product \ + ) VALUES ( \ + '" + batchType + "', \ + " + str(stageID) + ", \ + '" + config.survey + "', \ + '" + config.dvoLabel + "', \ + '" + config.datastoreProduct + "' \ + )" + + self.execute(sql) + + sql = "SELECT MAX(batch_id) FROM batch" + + try: + rs = self.executeQuery(sql) + rs.first() + batchID = rs.getInt(1) + self.logger.debug("Created new batch in ippToPsps database with batchID = %d" % batchID) + except: + self.logger.exception("Unable to get batch ID") + batchID = -1 + + return batchID; @@ -346,4 +694,326 @@ self.execute(sql) + ''' + Resets a batch ready for re-loading to the datastore + ''' + def resetBatch(self, batchID): + + sql = "UPDATE batch SET \ + loaded_to_datastore = 0, \ + loaded_to_ODM = 0, \ + merge_worthy = 0, \ + merged = 0, \ + deleted_datastore = 0, \ + deleted_dxlayer = 0, \ + comment = 'This batch was reset, ready for re-load to datastore' \ + WHERE batch_id = " + str(batchID) + + self.execute(sql) + + ''' + Remove client + ''' + def removeClient(self, host, pid): + self.execute("DELETE FROM clients WHERE host = '" + host + "' AND pid = " + str(pid)) + + ''' + Insert client + ''' + def insertClient(self, type, host, pid, config): + self.execute("INSERT INTO clients (timestamp, type, host, pid, config) VALUES (now(), '" + type + "', '" + host + "', " + str(pid) + ", '" + config + "')") + + ''' + Update clients, or inserts it for the first time + ''' + def updateClient(self, type, host, pid, config): + + try: + self.insertClient(type, host, pid, config) + except: + self.execute("UPDATE clients SET timestamp = now() WHERE host = '" + host + "' AND pid = " + str(pid)) + + ''' + Is this client paused? + ''' + def isPaused(self, host, pid): + return self.getClientsColumnBoolean("paused", host, pid) + + ''' + Is this client killed? + ''' + def isKilled(self, host, pid): + return self.getClientsColumnBoolean("killed", host, pid) + + ''' + Returns boolean value for this column in clients table + ''' + def getClientsColumnBoolean(self, col, host, pid): + + sql = "SELECT " + col + " \ + FROM clients \ + WHERE host = '" + host + "' AND pid = " + str(pid) + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) == 1: return True + else: return False + except: + self.logger.exception("Unable to check whether " + col + " is true or false in clients table") + + return False + + + ''' + Populates Config object + ''' + def readConfig(self, host, pid): + + # first get config defined for this client + sql = "SELECT config \ + FROM clients \ + WHERE host = '" + host + "' \ + AND pid = " + str(pid) + + try: + rs = self.executeQuery(sql) + rs.first() + self.config.name = rs.getString(1) + except: + self.logger.errorPair("No config set for", "%s (pid=%d)" % (host, pid)) + self.config.isLoaded = False + return False + + # now load that config + sql = "SELECT \ + datastore_product, \ + datastore_type, \ + datastore_publish, \ + dvo_label, \ + dvo_location, \ + min_ra, \ + max_ra, \ + min_dec, \ + max_dec, \ + box_size, \ + base_path, \ + data_release, \ + delete_local, \ + delete_datastore, \ + delete_dxlayer, \ + epoch, \ + survey, \ + psps_survey, \ + queue_P2, \ + queue_ST \ + FROM config \ + WHERE name = '" + self.config.name + "'" + + self.config.batchTypes = [] + try: + rs = self.executeQuery(sql) + rs.first() + self.config.datastoreProduct = rs.getString(1) + self.config.datastoreType = rs.getString(2) + if rs.getInt(3) == 1: self.config.datastorePublishing = True + else: self.config.datastorePublishing = False + self.config.dvoLabel = rs.getString(4) + self.config.dvoLocation = rs.getString(5) + self.config.minRa = rs.getDouble(6) + self.config.maxRa = rs.getDouble(7) + self.config.minDec = rs.getDouble(8) + self.config.maxDec = rs.getDouble(9) + + self.config.boxSize = rs.getDouble(10) + self.config.halfBox = self.config.boxSize/2.0 + self.config.boxSizeWithBorder = self.config.boxSize + (self.config.BORDER * 2) + + self.config.basePath = rs.getString(11) + self.config.dataRelease = rs.getInt(12) + self.config.deleteLocal = rs.getInt(13) + self.config.deleteDatastore = rs.getInt(14) + self.config.deleteDxLayer = rs.getInt(15) + self.config.epoch = rs.getString(16) + self.config.survey = rs.getString(17) + self.config.pspsSurvey = rs.getString(18) + if rs.getInt(19) == 1: self.config.batchTypes.append("P2") + if rs.getInt(20) == 1: self.config.batchTypes.append("ST") + self.config.force = False # TODO + self.config.test = False # TODO + self.config.isLoaded = True + except: + self.logger.errorPair("Could not read config with name", self.config.name) + self.config.isLoaded = False + + return self.config.isLoaded + + ''' + Gets id of box with these coords + ''' + def getBoxId(self, config, ra, dec, boxSide): + + sql = "SELECT id FROM box \ + WHERE config = '" + config + "' \ + AND ra_center = " + str(ra) + " \ + AND dec_center = " + str(dec) + " \ + AND box_side = " + str(boxSide) + + id = -1 + try: + rs = self.executeQuery(sql) + rs.first() + id = rs.getInt(1) + except: + pass + + return id + + + ''' + Remove all boxes and associated ids in pending table + ''' + def removeAllBoxes(self, config): + self.execute("DELETE FROM box WHERE config = '" + config + "'") + + ''' + Removes a box and it's associated ids in the pending table + ''' + def removeBox(self, config, ra, dec, boxSide): + self.execute("DELETE FROM box \ + WHERE config = '" + config + "' \ + AND ra_center = " + str(ra) + " \ + AND dec_center = " + str(dec) + " \ + AND box_side = " + str(boxSide)) + + ''' + Inserts new box into box table. If it already exisits, it returns existing box id + ''' + def insertBox(self, config, ra, dec, boxSide): + + id = self.getBoxId(config, ra, dec, boxSide) + + # does not already exists so insert + if id < 0: + + sql = "INSERT INTO box \ + (config, ra_center, dec_center, box_side) \ + VALUES \ + ('" + config + "', " + str(ra) + ", " + str(dec) + ", " + str(boxSide) + ")" + + self.execute(sql) + id = self.getBoxId(config, ra, dec, boxSide) + + return id + + ''' + Inserts new box ids into the pending table + ''' + def insertPending(self, box_id, batchType, ids): + + # first delete old pending items + self.execute("DELETE FROM pending WHERE box_id = " + str(box_id)) + + for id in ids: + sql = "INSERT INTO pending \ + (box_id, batch_type, stage_id) \ + VALUES \ + (" + str(box_id) + ", '" + batchType + "', " + str(id) + ")" + + self.execute(sql) + + ''' + Returns ids for all boxes with pending items + ''' + def getBoxIds(self, config): + + sql = "SELECT DISTINCT id \ + FROM box \ + JOIN pending ON (id = box_id) \ + WHERE config = '" + config + "' \ + ORDER BY ra_center, dec_center" + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.errorPair("Can't get boxes", sql) + + return ids + + ''' + Returns dimensions of this box + ''' + def getBoxDimensions(self, boxId): + + sql = "SELECT ra_center, dec_center, box_side FROM box WHERE id = " + str(boxId) + + boxDim = {} + rs = self.executeQuery(sql) + rs.first() + + boxDim['RA'] = rs.getFloat(1) + boxDim['DEC'] = rs.getFloat(2) + boxDim['SIDE'] = rs.getFloat(3) + + return boxDim + + ''' + Returns ids for pending items in this box for this batch type + ''' + def getPendingIds(self, boxId, batchType): + + sql = "SELECT stage_id FROM pending WHERE batch_type = '" + batchType + "' AND box_id = " + str(boxId) + + ids = [] + try: + rs = self.executeQuery(sql) + while (rs.next()): ids.append(rs.getInt(1)) + rs.close() + except: + self.logger.errorPair("Can't get ids from this box", sql) + + return ids + + ''' + Returns max number of pensing items in a box for this config + ''' + def getMaxPendingInBox(self, config): + + sql = "SELECT MAX(pending) \ + FROM (SELECT COUNT(stage_id) AS pending \ + FROM pending \ + JOIN box ON (id = box_id) \ + WHERE config = '" + config + "' \ + GROUP BY box_id) AS t" + + try: + rs = self.executeQuery(sql) + rs.first() + return rs.getInt(1) + except: + self.logger.errorPair("Could not count max pending", sql) + return 100 + + ''' + Writes density-plot data to file for pending stuff over sky for given config + ''' + def createPendingDensityPlotData(self, config, DATFILE): + + sql = "SELECT ra_center, dec_center, COUNT(stage_id) \ + FROM box \ + LEFT JOIN pending ON (id = box_id) \ + WHERE config = '" + config + "' \ + GROUP BY id \ + ORDER BY ra_center, dec_center" + + try: + rs = self.executeQuery(sql) + while (rs.next()): + print >> DATFILE, rs.getFloat(1), rs.getFloat(2), rs.getInt(3) + rs.close() + except: + self.logger.errorPair("Can't get density plot data", sql) ''' @@ -352,5 +1022,5 @@ def __del__(self): - self.logger.debug("IppToPspsDb Desstructor") + self.logger.debug("IppToPspsDb Destructor") self.stmt.close() self.con.close() Index: branches/meh_branches/ppstack_test/ippToPsps/jython/loader.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/loader.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/loader.py (revision 33415) @@ -0,0 +1,219 @@ +#!/usr/bin/env jython + +# +# The main loading program for ippToPsps. See usage below. +# + +import time +import sys +import os +import traceback +import logging.config + +from ipptopsps import IppToPsps +from scratchdb import ScratchDb +from gpc1db import Gpc1Db +from datastore import Datastore +from batch import Batch +from dvo import Dvo + +from initbatch import InitBatch +from stackbatch import StackBatch +from detectionbatch import DetectionBatch + +''' +Loader class +''' +class Loader(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Loader, self).__init__(argv) + + # create gpc1 database objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + + # connect to scratch database + scratchDbs = ['ipptopsps_scratch', 'ipptopsps_scratch2', 'ipptopsps_scratch3'] + for dbName in scratchDbs: + self.scratchDb = ScratchDb(self.logger, self.config, dbName) + if self.scratchDb.anyOtherConnections(): + self.logger.errorPair("This scratch Db is already in use", dbName) + self.scratchDb.disconnect() + continue + self.logger.infoPair("Using scratch Db", self.scratchDb.dbName) + break + + if not self.scratchDb.connected: + self.logger.errorPair("Cannot connect to a", "scratch database") + raise Exception("No scratch Db") + + # now pass scratch Db to dvo object + self.dvo.setScratchDb(self.scratchDb.dbName) + + # if an IN batch is requested, create and quit + if len(sys.argv) > 2 and sys.argv[2] == "init": + batchID = self.ippToPspsDb.createNewBatch("IN", 0, self.config) + if batchID > 0: + batch = InitBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + batchID) + batch.run() + + self.exitProgram("init batch created") + + + # set a poll time of about 1 minutes + self.parsePollTimeArg("0.0166") + + self.config.printAll() + + ''' + Overrides base-class version + ''' + def refreshConfig(self): + + ret = super(Loader, self).refreshConfig() + self.dvo = Dvo(self.logger, self.config) + try: self.dvo.setScratchDb(self.scratchDb.dbName) + except: pass + self.datastore = Datastore(self.logger, self.config, self.ippToPspsDb) + + return ret + + ''' + Main processing loop + ''' + def run(self): + + while True: + + boxIds = self.ippToPspsDb.getBoxIds(self.config.name) + + self.logger.infoPair("Boxes with unprocessed data", "%d" % len(boxIds)) + + for boxId in boxIds: + + self.checkClientStatus() + + # get box info. if boxes have changed, break and start again + try: + boxDim = self.ippToPspsDb.getBoxDimensions(boxId) + except: + self.logger.infoPair("Can't get details for this box", "%d" % boxId) + break + + for batchType in self.config.batchTypes: + + ids = self.ippToPspsDb.getPendingIds(boxId, batchType) + + if len(ids) < 1: + self.logger.debugPair("No " + batchType + " items found in this box", "skipping") + continue + + self.logger.infoSeparator() + self.logger.infoPair("Box dimensions", "%.1f / %.1f / %.1f" % (boxDim['RA'], boxDim['DEC'], boxDim['SIDE'])) + self.logger.infoPair(batchType + " items found in this box", "%d" % len(ids)) + boxSizeWithBorder = boxDim['SIDE'] + (self.config.BORDER * 2) + + # look in DVO for this box (with extra border) + self.dvo.setSkyAreaAsBox(boxDim['RA'], boxDim['DEC'], boxSizeWithBorder) + sizeToBeIngested = self.dvo.getDiskSizeOfRegionsToBeIngested() + if sizeToBeIngested == 0.0: smfsPerGB = 999999999 + else: smfsPerGB = len(ids)/sizeToBeIngested + self.logger.infoPair("DVO to be ingested", "%.1f GB" % sizeToBeIngested) + self.logger.infoPair("smfs-per-GB", "%.1f" % smfsPerGB) + + # should do we pre-ingest stuff from DVO? + if smfsPerGB > 40: + if not self.dvo.sync(): + self.logger.errorPair("Could not sync DVO with MySQL", "skipping") + continue + + useFullTables = 1 + else: useFullTables = 0 + + self.logger.infoBool("Using pre-ingested DVO data?", useFullTables) + self.processTheseItems(batchType, ids, useFullTables) + + self.checkClientStatus() + if not self.waitForPollTime(): break + + + ''' + Actually loops through items and processes them + ''' + def processTheseItems(self, batchType, ids, useFullTables): + + self.logger.infoPair("%s items queued" % batchType, "%d" % len(ids)) + + # loop round IDs of all items to be processed + self.ippToPspsDb.lockBatchTable() + unattemptedCount = 0 + for id in ids: + + batchID = self.ippToPspsDb.createNewBatch(batchType, id, self.config) + + if batchID < 0: + unattemptedCount += 1 + continue + + self.ippToPspsDb.unlockTables() + + if batchType == "P2": + batch = DetectionBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + elif batchType == "ST": + batch = StackBatch(self.logger, + self.config, + self.gpc1Db, + self.ippToPspsDb, + self.scratchDb, + id, + batchID, + useFullTables) + + batch.run() + + # check that we should continue + self.checkClientStatus() + + self.ippToPspsDb.lockBatchTable() + + self.logger.infoSeparator() + + # if in test mode, then quit after one batch + if self.config.test: break + + self.ippToPspsDb.unlockTables() + self.logger.infoPair("Unattempted batches", "%d" % unattemptedCount) + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Cleanup, self).printUsage() + print " [init]" + + +''' +Start of program. +''' +try: + loader = Loader(sys.argv) + loader.run() + loader.exitProgram("finished") +except Exception, e: + print str(e) + traceback.print_exc() Index: branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/metrics.py (revision 33415) @@ -1,126 +1,174 @@ import math +import time import logging import smtplib import sys import os +import datetime -from java.lang import Math -from org.apache.commons.math.special import Erf -from xml.etree.ElementTree import ElementTree, Element, tostring - -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb +from ipptopsps import IppToPsps +from czardb import CzarDb from gpc1db import Gpc1Db ''' -Makes a gnuplot of the data +Metrics class ''' -def plotMe(batchType, file): +class Metrics(IppToPsps): - OUTPUTFILE = "plots/" + DVOLABEL + "_" + batchType + ".png" - f=os.popen('gnuplot', 'w') + ''' + Constructor + ''' + def __init__(self, argv): + super(Metrics, self).__init__(argv) - if 0: - TERM = "X11" - else: - TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" + if len(sys.argv) > 2: self.parsePollTimeArg(sys.argv[2]) + + # create database objects + self.czarDb = CzarDb(self.logger, self.config) + self.gpc1Db = Gpc1Db(self.logger, self.config) + + ''' + Run... + ''' + def run(self): + + while True: + + self.logger.infoTitle("ippToPsps loading summary") + now = datetime.datetime.now() + self.logger.infoPair("Time now", now.strftime("%Y-%m-%d %H:%M:%S")) + self.logger.infoPair("Loading epoch", self.config.epoch) + self.logger.infoPair("DVO label", self.config.dvoLabel) + self.config.printBoxCoords() - print >> f, "set term " + TERM + "; \ - set output \"" + OUTPUTFILE + "\"; \ - set title \"ippToPsps : " + batchType + " loading for " + DVOLABEL + "\"; \ - set boxwidth; \ - set xtic rotate by -90 scale 0; \ - set style data histogram; \ - set style histogram rowstacked; \ - set style fill solid border -1; \ - set ylabel \"Items processed per loading stage\"; \ - set boxwidth 0.75; \ - plot '" + file + "' using 2:xtic(1) notitle lt 2, '' using 3 notitle lt 1, '' using 4 notitle lt rgb \"#808080\"" + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + self.logger.info("|Type| batches per hour | last 24 hours | per day this week | total detections | last published |") + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + rateP2 = self.printStats("P2") + rateST = self.printStats("ST") + self.logger.info("+----+------------------+---------------+-------------------+------------------+----------------+") + + self.logger.info("") + + stages = self.ippToPspsDb.getStages() + colCount = len(stages) + self.writeTableSeparator(colCount) + sys.stdout.write("|Type| DVO ") + + for stage in stages: sys.stdout.write("|%19s" % stage) + sys.stdout.write("|\n") + sys.stdout.write("| | ") + for stage in stages: sys.stdout.write("| Pend Succ Fail ") + sys.stdout.write("|\n") + self.writeTableSeparator(colCount) + pendP2Processed = self.printTableRow(stages, "P2") + pendSTProcessed = self.printTableRow(stages, "ST") + self.writeTableSeparator(colCount) - f.flush() + self.logger.info("") + try: self.logger.infoPair("Estimated time for P2s", "%.1f hours" % (pendP2Processed / rateP2)) + except: pass + try: self.logger.infoPair("Estimated time for STs", "%.1f hours" % (pendSTProcessed / rateST)) + except: pass + + self.checkClientStatus() + if not self.waitForPollTime(): break -''' -Prepares a string of the format: -n-processed(n-pending) + ''' + Formats an int to a string + ''' + def getIntAsString(self, value): + if value == 0: return "" + return str(value) + + ''' + Prints (to the log) one row of data (one row being one batch-type) + ''' + def printTableRow(self, stages, batchType): + + # get a master list of IDs in DVO for this batch type + #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec) + allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType) + prevList = allIDs + numEverything = len(allIDs) + + sys.stdout.write("| %2s |%6d " % (batchType, numEverything)) + + tempFilename = batchType + '.dat' + DATFILE = open(tempFilename,'w') + print >> DATFILE, "DVO", numEverything, 0, 0 + for stage in stages: + + # get lists. Use unions with prev list to make sure the right stuff is included + success = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, 1)) & set(prevList)) + fail = list(set(self.ippToPspsDb.getStageIDs(batchType, self.config.epoch, self.config.dvoLabel, stage, -1)) & set(prevList) - set(success)) + pending = list(set(prevList) - set(success) - set(fail)) + + # count stuff + numSuccess = len(success) + numFail = len(fail) + numPending = len(pending) + + #if stage == 'loaded_to_datastore' and batchType == "P2": + # for n in fail: print n + + str = "\033[1;34m%5s\033[1;m \033[1;32m%5s\033[1;m \033[1;31m%5s\033[1;m " % (self.getIntAsString(numPending), self.getIntAsString(numSuccess), self.getIntAsString(numFail)) + sys.stdout.write("|%-19s" % str) + + prevList = success + print >> DATFILE, stage, numSuccess, numFail, numPending + + self.czarDb.insertStats(stage, self.config.dvoLabel, batchType, numPending, numSuccess, numFail) + + if stage == 'processed': numPendingProcessed = numPending + + sys.stdout.write("|\n") + DATFILE.close() + + return numPendingProcessed + + ''' + Writes table separator + ''' + def writeTableSeparator(self, colCount): + + sys.stdout.write("+----+-------") + for i in range(0, colCount): sys.stdout.write("+-------------------") + sys.stdout.write("+\n") + + ''' + Prints various timing stats for this batch type + ''' + def printStats(self, batchType): + + rate = self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 HOUR") / 1.0 + + self.logger.info("| %2s | %16.1f | %13d | %17.1f | %16d | %14s |" % + (batchType, + rate, + self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "24 HOUR"), + self.ippToPspsDb.countBatchesInLastPeriod(batchType, self.config.epoch, self.config.dvoLabel, "1 WEEK") / 7.0, + self.ippToPspsDb.getTotalDetectionsPublished(batchType, self.config.epoch, self.config.dvoLabel), + self.ippToPspsDb.getTimeOfLastBatchPublished(batchType, self.config.epoch, self.config.dvoLabel))) + + return rate + -''' -def prepareStr(masterList, prevList, thisList, FILE, name): + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Metrics, self).printUsage() + print " [hours]" - unsuccessful = list(set(prevList) - set(thisList)) - countTotal = len(masterList) - countSuccessful = len(thisList) - countUnsuccessful = len(unsuccessful) - - str = "%d" % countSuccessful - if countUnsuccessful: str = str + "(%d)" % countUnsuccessful - - print >> FILE, name, countSuccessful, countUnsuccessful, (countTotal - countSuccessful - countUnsuccessful) - #;sys.stdout.softspace=0; - return str - -''' -Prints (to the log) one row of data (one row being one batch-type) -''' -def printTableRow(batchType): - - allIDs = gpc1Db.getIDsInThisDVODbForThisStage(DVOLABEL, batchType) - processedIDs = ippToPspsDb.getProcessedIDs(batchType, EPOCH, DVOLABEL) - datastoreIDs = ippToPspsDb.getloadedToDatastoreIDs(batchType, EPOCH, DVOLABEL) - loadedToODMIDs = ippToPspsDb.getLoadedToODMIDs(batchType, EPOCH, DVOLABEL) - mergeWorthyIDs = ippToPspsDb.getMergeWothyIDs(batchType, EPOCH, DVOLABEL) - mergedIDs = ippToPspsDb.getMergedIDs(batchType, EPOCH, DVOLABEL) - - tempFilename = batchType + '.dat' - DATFILE = open(tempFilename,'w') - prepareStr(allIDs, allIDs, allIDs, DATFILE, "inDVO"), - - logger.info("| %2s | %-12d | %-12s | %-12s | %-12s | %-12s | %-12s | %-12s | %-19s |" % ( - batchType, - len(allIDs), - prepareStr(allIDs, allIDs, processedIDs, DATFILE, "processed"), - prepareStr(allIDs, processedIDs, datastoreIDs, DATFILE, "onDatastore"), - prepareStr(allIDs, datastoreIDs, loadedToODMIDs, DATFILE, "loadedToODM"), - prepareStr(allIDs, loadedToODMIDs, mergeWorthyIDs, DATFILE, "MergeWorthy"), - prepareStr(allIDs, mergeWorthyIDs, mergedIDs, DATFILE, "Merged"), - len(list(set(allIDs) - set(mergedIDs))), - ippToPspsDb.getTimeOfLastBatchPublished(batchType, EPOCH, DVOLABEL))) - - plotMe(batchType, tempFilename) - DATFILE.close() - #os.remove(tempFilename) ''' Start of program. ''' -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "** Usage: " + sys.argv[0] + " " - sys.exit(1) - -# open config file -configDoc = ElementTree(file=CONFIG) - -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("metrics") -logger.setup(configDoc, "metrics") - -ippToPspsDb = IppToPspsDb(logger, configDoc) -gpc1Db = Gpc1Db(logger, configDoc) - -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text - -logger.infoTitle("ippToPsps loading summary") -logger.infoPair("Loading epoch", EPOCH) -logger.infoPair("DVO label", DVOLABEL) -logger.info("") -logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+") -logger.info("| Type | Total in DVO | Processed | On datastore | Loaded to ODM | Merge-worthy | Merged | Total missing | Last published |") -logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+") -printTableRow("P2") -printTableRow("ST") -logger.info("+------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------------+") +metrics = Metrics(sys.argv) +metrics.run() +metrics.exitProgram("finished") Index: branches/meh_branches/ppstack_test/ippToPsps/jython/mysql.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/mysql.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/mysql.py (revision 33415) @@ -21,16 +21,17 @@ ''' - def __init__(self, logger, doc, dbType): + def __init__(self, logger, config, dbType, dbName=None): # set up logging self.logger = logger - self.doc = doc + self.config = config self.logger.debug("MySql class constructor") # open config and grab database parameters - self.dbName = self.doc.find(dbType +"/name").text - self.dbHost = self.doc.find(dbType +"/host").text - self.dbUser = self.doc.find(dbType +"/user").text - self.dbPass = self.doc.find(dbType +"/password").text + if not dbName: self.dbName = config.getDbName(dbType) + else: self.dbName = dbName + self.dbHost = config.getDbHost(dbType) + self.dbUser = config.getDbUser(dbType) + self.dbPass = config.getDbPassword(dbType) # set up JDBC connection @@ -42,9 +43,10 @@ except: self.logger.error("Unable to connect to " + self.url) - self.everythingOK = False + self.everythingOK = False # TODO need this? + self.connected = False return self.everythingOK = True - + self.connected = True ''' @@ -53,4 +55,5 @@ def disconnect(self): self.con.close() + self.connected = False ''' @@ -61,4 +64,16 @@ self.logger.debug("MySql destructor") self.disconnect() + + ''' + Locks a table. This is wait if the locks is already held by another session + ''' + def lockTable(self, table): + self.execute("LOCK TABLES " + table + " WRITE") + + ''' + Unlocks all tables locked by this session. + ''' + def unlockTables(self): + self.execute("UNLOCK TABLES") ''' @@ -69,9 +84,28 @@ connectionID = self.getLastConnectionID() if connectionID == self.connectionID: - self.logger.error("NOT going to kill THIS connection ID") + self.logger.debug("NOT going to kill THIS connection ID") return sql = "KILL %d" % connectionID self.execute(sql) + + ''' + Checks whether there are any other connections to this database + ''' + def anyOtherConnections(self): + + sql = "SELECT COUNT(*) \ + FROM INFORMATION_SCHEMA.PROCESSLIST \ + WHERE DB='" + self.dbName + "'" + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 1: return True + else: return False + except: + self.logger.errorPair("Could not check for connection", sql) + pass + + return True ''' @@ -97,4 +131,24 @@ ''' + OPTIMIZEs a table + ''' + def optimizeTable(self, table): + + sql = "OPTIMIZE TABLE " + table + try: self.execute(sql) + except: return + + ''' + Creates a MySQL-friendly table name from the provided string + ''' + def getDbFriendlyTableName(self, origName): + + name = origName + name = name.replace('.', '_') + name = name.replace('/', '_') + + return name + + ''' Drops a table ''' @@ -215,7 +269,9 @@ nBad = rs.getInt(1) - sql="DELETE from " + tableName + " WHERE " + columnName + " = " + value + sql="DELETE FROM " + tableName + " WHERE " + columnName + " = " + value self.execute(sql) - self.logger.info("%s '%s' values in %s %5d deleted" % (columnName, value, tableName, nBad)) + self.logger.debugPair("%s rows with %s" % (columnName, value), "%d deleted" % nBad) + + return nBad ''' @@ -229,7 +285,7 @@ nBad = rs.getInt(1) - sql="DELETE from " + tableName + " WHERE " + columnName + " IS NULL" + sql="DELETE FROM " + tableName + " WHERE " + columnName + " IS NULL" self.execute(sql) - self.logger.debugPair("NULL %s values" % columnName, "%d deleted" % nBad) + self.logger.debugPair("NULL %s values in %s" % (columnName, tableName), "%d deleted" % nBad) return nBad Index: branches/meh_branches/ppstack_test/ippToPsps/jython/odm.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/odm.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/odm.py (revision 33415) @@ -2,4 +2,6 @@ import re +import os +import signal import sys import tempfile @@ -33,7 +35,8 @@ p = Popen("../perl/checkOdmStatus.pl -b " + batch + " -f " + tempFile.name, shell=True, - stdout=PIPE) + stdout=PIPE, + close_fds=True) p.wait() - + if p.returncode != 0: self.logger.debugPair("ODM polling", "failed") @@ -45,4 +48,6 @@ tempFile.close() + p.stdout.close() + if not ret: self.logger.debugPair("Cannot read ODM file for batch", "%d" % batchID) @@ -58,5 +63,4 @@ loadedToODM = 0 - loadFailed = 0 mergeWorthy = 0 merged = 0 @@ -66,14 +70,26 @@ STATE = doc.find("{%s}OdmBatchState/{%s}BatchState" % (NAMESPACE, NAMESPACE)).text MESSAGE = doc.find("{%s}OdmBatchState/{%s}Message" % (NAMESPACE, NAMESPACE)).text + results['MESSAGE'] = MESSAGE + results['DETAILS'] = "None" - if re.search("failed", STATE): loadFailed = 1 - if re.search("MergeWorthy", MESSAGE): mergeWorthy = 1 - if re.search("Merge[1-9]Completed", MESSAGE): merged = 1 - if mergeWorthy or merged or loadFailed: loadedToODM = 1 + # we assume that batches 'OnHold' are failed as it is very rare for such + # batches to have there state changed + if re.search("failed", STATE) or re.search("OnHold", MESSAGE): + loadedToODM = -1 + DETAILS = doc.find("{%s}OdmBatchState/{%s}Details" % (NAMESPACE, NAMESPACE)).text + results['DETAILS'] = DETAILS + if re.search("processing", STATE): + loadedToODM = 1 + if re.search("MergeWorthy", MESSAGE): + loadedToODM = 1 + mergeWorthy = 1 + if re.search("Merge[1-9]Completed", MESSAGE): + loadedToODM = 1 + mergeWorthy = 1 + merged = 1 except: - return 0 + pass results['LOADEDTOODM'] = loadedToODM - results['LOADFAILED'] = loadFailed results['MERGEWORTHY'] = mergeWorthy results['MERGED'] = merged Index: branches/meh_branches/ppstack_test/ippToPsps/jython/plotter.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/plotter.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/plotter.py (revision 33415) @@ -0,0 +1,70 @@ +#!/usr/bin/env jython + +import tempfile +import os +import sys + +from ipptopsps import IppToPsps + + +''' +Plotter class +''' +class Plotter(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Plotter, self).__init__(argv) + + ''' + Runs... + ''' + def run(self): + + tempFilename = "./plotData.dat" + DATFILE = open(tempFilename,'w') + self.ippToPspsDb.createPendingDensityPlotData(self.config.name, DATFILE) + DATFILE.close() + + OUTPUTFILE = self.config.name + ".png" + f = os.popen('gnuplot', 'w') + + max = self.ippToPspsDb.getMaxPendingInBox(self.config.name) + + if 0: + TERM = "X11" + else: + TERM = "png font \"/usr/share/fonts/corefonts/arial.ttf\" 8" + + print >> f, "set term " + TERM + "; \ + set output \"" + OUTPUTFILE + "\"; \ + set title \"Density plot of unprocessed stuff\"; \ + set grid; \ + set xrange [" + str(self.config.minRa) + ":" + str(self.config.maxRa) + "] reverse; \ + set yrange [" + str(self.config.minDec) + ":" + str(self.config.maxDec) + "]; \ + unset key; \ + set palette rgb 22,13,10; \ + set ylabel \"Dec\"; \ + set ylabel \"RA\"; \ + set cbrange [0:" + str(max) + "]; \ + set datafile missing \"NaN\"; \ + plot '" + tempFilename + "' using 1:2:3 with image" + + f.flush() + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(Plotter, self).printUsage() + print " []" + + +''' +Start of program +''' +plotter = Plotter(sys.argv) +plotter.run() +plotter.exitProgram("finished") Index: branches/meh_branches/ppstack_test/ippToPsps/jython/pollOdm.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/pollOdm.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/pollOdm.py (revision 33415) @@ -6,56 +6,133 @@ import re import sys +import time -import logging.config -from pslogger import PSLogger -from ipptopspsdb import IppToPspsDb -from gpc1db import Gpc1Db +from ipptopsps import IppToPsps +from config import Config from odm import Odm from batch import Batch -if len(sys.argv) > 1: CONFIG = sys.argv[1] -else: - print "\n** Usage: " + sys.argv[0] + " \n" - sys.exit(1) +''' +PollOdm class +''' +class PollOdm(IppToPsps): -# open config file -configDoc = ElementTree(file=CONFIG) + ''' + Constructor + ''' + def __init__(self, argv): + super(PollOdm, self).__init__(argv) -# set up logging -logging.setLoggerClass(PSLogger) -logger = logging.getLogger(sys.argv[0]) -logger.setup(configDoc, sys.argv[0]) + # get batch and stage args + if len(sys.argv) > 3: + self.BATCHTYPE = sys.argv[2] + self.STAGE = sys.argv[3] + else: + self.printUsage() + self.exitProgram("incorrect args") -ippToPspsDb = IppToPspsDb(logger, configDoc) -gpc1Db = Gpc1Db(logger, configDoc) -odm = Odm(logger, configDoc) + # get poll time arg + if len(sys.argv) > 4: self.parsePollTimeArg(sys.argv[4]) -DVOLABEL = configDoc.find("dvo/gpc1Label").text -EPOCH = configDoc.find("options/epoch").text + if self.STAGE == "all": self.stages = ['unloaded', 'unmergeworthy', 'unmerged'] + else: self.stages = [self.STAGE] -ids = ippToPspsDb.getProcessedButUnmergedBatchIDs(EPOCH, DVOLABEL) + if self.BATCHTYPE == "all": self.batchTypes = ['P2', 'ST'] + else: self.batchTypes = [self.BATCHTYPE] -logger.infoPair("Processed but un-merged batches", "%d" % len(ids)) + for stage in self.stages: self.logger.infoPair("Checking stage", stage) + for batchType in self.batchTypes: self.logger.infoPair("Checking batch type", batchType) + ''' + Overrides base-class version + ''' + def refreshConfig(self): + ret = super(PollOdm, self).refreshConfig() + self.odm = Odm(self.logger, self.config) + return ret -results = {} + ''' + Run... + ''' + def run(self): -logger.info("+-----------+----------------+----------------+----------------+----------------+") -logger.info("| Batch | Loaded to ODM | Load failed | Merge worthy | Merged |") -logger.info("+-----------+----------------+----------------+----------------+----------------+") -for id in ids: + while True: - if odm.checkBatch(id, results): - ippToPspsDb.updateOdmStatus(id, results) - logger.info("| %9s | %14d | %14d | %14d | %14d |" % - (Batch.getNameFromID(id), - results['LOADEDTOODM'], - results['LOADFAILED'], - results['MERGEWORTHY'], - results['MERGED'])) - else: - logger.info("| %9s | failed to poll ODM |" % Batch.getNameFromID(id)) + for stage in self.stages: + for batchType in self.batchTypes: + self.poll(batchType, stage) -logger.info("+-----------+----------------+----------------+----------------+----------------+") + self.checkClientStatus() + if not self.waitForPollTime(): break + ''' + Method to actually poll ODM + ''' + def poll(self, batchType, stage): + + if stage == "unloaded": + ids = self.ippToPspsDb.getUnloadedBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + elif stage == "unmergeworthy": + ids = self.ippToPspsDb.getUnmergeWorthyBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + elif stage == "unmerged": + ids = self.ippToPspsDb.getUnmergedBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + + self.logger.infoTitle("Checking %d %s %s batches" % (len(ids), stage, batchType)) + + results = {} + newLoaded = 0 + newMergeWorthy = 0 + newMerged = 0 + + self.logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+") + self.logger.info("| Batch | Loaded to ODM | Merge worthy | Merged | New loaded |New merge worthy| New merged |") + self.logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+") + for id in ids: + + if self.odm.checkBatch(id, results): + self.ippToPspsDb.updateOdmStatus(id, results) + + if stage == "unloaded": + if results['LOADEDTOODM'] == 1: newLoaded = newLoaded + 1 + if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 + if results['MERGED'] == 1: newMerged = newMerged + 1 + + elif stage == "unmergeworthy": + if results['MERGEWORTHY'] == 1: newMergeWorthy = newMergeWorthy + 1 + if results['MERGED'] == 1: newMerged = newMerged + 1 + + elif stage == "unmerged": + if results['MERGED'] == 1: newMerged = newMerged + 1 + + self.logger.info("| %9s | %14d | %14d | %14d | %14d | %14d | %14d |" % + (Batch.getNameFromID(id), + results['LOADEDTOODM'], + results['MERGEWORTHY'], + results['MERGED'], + newLoaded, + newMergeWorthy, + newMerged)) + + + else: + self.logger.info("| %9s | failed to poll ODM |" % Batch.getNameFromID(id)) + + self.logger.info("+-----------+----------------+----------------+----------------+----------------+----------------+----------------+") + + + ''' + Overrides base-class version + ''' + def printUsage(self): + super(PollOdm, self).printUsage() + print " [hours]" + + + +''' +Program starts here +''' +pollodm = PollOdm(sys.argv) +pollodm.run() +pollodm.exitProgram("finished") Index: branches/meh_branches/ppstack_test/ippToPsps/jython/pslogger.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/pslogger.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/pslogger.py (revision 33415) @@ -3,11 +3,13 @@ import os import sys +import socket class PSLogger(logging.getLoggerClass()): ''' - Sets up logging using values from the ippToPsps config file + Sets up logging using provided values and this process hostname and PID if sending to file + This will default to stout and no file output ''' - def setup(self, doc, name, stdout=1, sendToFile=0): + def setup(self, programName, basePath, configName, host, pid, stdout=1, sendToFile=0): formatter = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', '%Y-%m-%d %H:%M:%S') @@ -15,9 +17,9 @@ if sendToFile: - PATH = doc.find("localOutPath").text + "/log" + + # generate path + PATH = basePath + "/log" if not os.path.exists(PATH): os.makedirs(PATH) - DVOLABEL = doc.find("dvo/gpc1Label").text - FULLPATH = PATH + "/" + name + "_" + DVOLABEL + ".log" - + FULLPATH = PATH + "/" + programName + "_" + configName + "_" + host + "_" + str(pid) + ".log" # opens file to be appended hdlr = logging.FileHandler(FULLPATH, "a") @@ -27,5 +29,5 @@ if stdout: stdout = logging.StreamHandler(sys.__stdout__) - #stdout.setFormatter(formatter) + stdout.setFormatter(formatter) self.addHandler(stdout) Index: branches/meh_branches/ppstack_test/ippToPsps/jython/queue.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/queue.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/queue.py (revision 33415) @@ -0,0 +1,137 @@ +#!/usr/bin/env jython + +# +# The main loading program for ippToPsps. See usage below. +# + +import time +import sys +import os +import logging.config + +from ipptopsps import IppToPsps +from gpc1db import Gpc1Db +from datastore import Datastore +from batch import Batch + +''' +Queue class +''' +class Queue(IppToPsps): + + ''' + Constructor + ''' + def __init__(self, argv): + super(Queue, self).__init__(argv) + + # create various objects + self.gpc1Db = Gpc1Db(self.logger, self.config) + + # set a poll time in hours + self.parsePollTimeArg("12") + + + ''' + Main processing loop. + Tiles area defined in config and looks for available stage_ids for each tile and writes them to ippToPsps database ready for processing + by one or more loading clients that may be running on any host + ''' + def run(self): + + # this outer while loop simply waits a few minutes then starts queuing again as more stuff may appear in DVO over time + while True: + + # queue up batches that are processed but not loaded to datastore + self.logger.infoTitle("Previous failed datastore loads") + for batchType in self.config.batchTypes: self.publishAnyUnpublishedBatches(batchType) + + # get totals for whole area to check if there is anything to do + processedIDs = self.ippToPspsDb.getProcessedIDs(batchType, self.config.epoch, self.config.dvoLabel) + consistantlyFailedIDs = self.ippToPspsDb.getConsistentlyFailedIDs(batchType, self.config.epoch, self.config.dvoLabel) + allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType, self.config.minRa, self.config.maxRa, self.config.minDec, self.config.maxDec) + #allIDs = self.gpc1Db.getIDsInThisDVODbForThisStage(self.config.dvoLabel, batchType) + ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs)) + + # loop through full range of RA/Dec queueing stuff in boxes of size self.config.boxSize + self.logger.info("+---------------------------------------------------------------------+") + self.logger.info("| All unprocessed items in area: %12d |" % len(ids)) + + if len(ids) > 0: + + self.logger.info("+-------------+-------------+-------------+-------------+-------------+") + self.logger.info("| RA | Dec | box size | in DVO | unprocessed |") + self.logger.info("+-------------+-------------+-------------+-------------+-------------+") + + # starting positions + ra = self.config.minRa + self.config.halfBox + dec = self.config.minDec + self.config.halfBox + + while ra <= self.config.maxRa: + while dec <= self.config.maxDec: + + self.checkClientStatus() + box_id = self.ippToPspsDb.insertBox(self.config.name, ra, dec, self.config.boxSize) + + # for each batch type look for items to queue in this box + for batchType in self.config.batchTypes: + + allIDs = self.gpc1Db.getIDsInThisDVODbForThisStageInThisBox( + self.config.dvoLabel, + batchType, + ra, + dec, + self.config.boxSize) + + if len(allIDs) < 1: + self.logger.info("| %5.1f | %5.1f | %9d | %9d | %9d |" % ( + ra, + dec, + self.config.boxSize, + len(allIDs), + 0)) + continue + + ids = list(set(allIDs) - set(processedIDs) - set(consistantlyFailedIDs)) + self.logger.info("| %5.1f | %5.1f | %9d | %9d | %9d |" % ( + ra, + dec, + self.config.boxSize, + len(allIDs), + len(ids))) + if len(ids) < 1: continue + + #self.logger.infoPair("write IDs to database", "NOW") + self.ippToPspsDb.insertPending(box_id, batchType, ids) + + dec = dec + self.config.boxSize + dec = self.config.minDec + self.config.halfBox + ra = ra + self.config.boxSize + + self.logger.info("+-------------+-------------+-------------+-------------+-------------+") + + if not self.waitForPollTime(): break + + ''' + Finds batches that processed ok, but did not get loaded to the datastore, then tries to load them + ''' + def publishAnyUnpublishedBatches(self, batchType): + + batchIDs = self.ippToPspsDb.getProcessedButFailedDatastoreBatchIDs(self.config.epoch, self.config.dvoLabel, batchType) + self.logger.infoPair("%s batches" % batchType, "%d" % len(batchIDs)) + + for batchID in batchIDs: + + batchName = Batch.getNameFromID(batchID) + subDir = Batch.getSubDir(self.config.basePath, batchType, self.config.dvoLabel) + tarballFile = Batch.getTarballFile(batchID) + self.logger.infoPair("Batch name", batchName) + + Batch.publishToDatastore(self.datastore, batchID, batchName, subDir, tarballFile) + +''' +Start of program. +''' +queue = Queue(sys.argv) +queue.run() +queue.exitProgram("finished") Index: branches/meh_branches/ppstack_test/ippToPsps/jython/regionTest.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/regionTest.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/regionTest.py (revision 33415) @@ -0,0 +1,96 @@ +from pslogger import PSLogger +from ipptopspsdb import IppToPspsDb +from gpc1db import Gpc1Db +from dvo import Dvo + +import sys +from xml.etree.ElementTree import ElementTree, Element, tostring +import logging + +''' +Start of program. +''' + +if len(sys.argv) > 1: CONFIG = sys.argv[1] +else: + print "** Usage: " + sys.argv[0] + " " + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +logging.setLoggerClass(PSLogger) +logger = logging.getLogger("regionTest") +logger.setup(configDoc, "regionTest") + + +gpc1Db = Gpc1Db(logger, configDoc) +dvo = Dvo(logger, configDoc) +#dvo.resetAllTables() + +DVOLABEL = configDoc.find("dvo/gpc1Label").text + +# get equatorial coord constraints, if any +try: + MINRA = float(configDoc.find("dvo/minRA").text) + MAXRA = float(configDoc.find("dvo/maxRA").text) + MINDEC = float(configDoc.find("dvo/minDec").text) + MAXDEC = float(configDoc.find("dvo/maxDec").text) +except: + MINRA = 0.0 + MAXRA = 360.0 + MINDEC = -30.0 + MAXDEC = 90.0 + +BORDER = 1.60 +#BOXSIZE = 4 +BOXSIZE = 1.0 +HALFBOX = BOXSIZE/2.0 + + + +ra = MINRA + HALFBOX +dec = MINDEC + HALFBOX + +while ra <= MAXRA: + while dec <= MAXDEC: + + logger.infoSeparator() + logger.infoPair("BOXSIZE", "%d" % BOXSIZE) + allIDs = gpc1Db.getIDsInThisDVODbForThisStageInThisBox(DVOLABEL, "P2", ra, dec, BOXSIZE) + logger.infoPair("Found", "%d exposures" % len(allIDs)) + + dvo.loadRegionInThisBoxWithBorder(ra, dec, BOXSIZE, BORDER) + + ''' + regionsToIngest = [] + regionsToPurge = [] + + dvo.getRegionsInThisBoxWithBorder(regionsToIngest, regionsToPurge, ra, dec, BOXSIZE, BORDER) + + size = dvo.getDiskSizeOfDvoRegions(regionsToIngest)/1073741824.0 + if size == 0: smfsPerGB = 0 + else: smfsPerGB = len(allIDs)/size + logger.infoPair("smf files per GB", "%f" % smfsPerGB) + ''' + dec = dec + BOXSIZE + break + ra = ra + BOXSIZE + break + +#regionsToIngest = [] +#regionsToPurge = [] +#dvo.getRegions(regionsToIngest, regionsToPurge) +#dvo.getRegions(regionsToIngest, regionsToPurge, MINRA-BORDER, MAXRA+BORDER, MINDEC-BORDER, MAXDEC+BORDER) +#size = dvo.getSizeOfDvoRegionFromFiles(files) +#logger.infoPair("Size of region in DVO", dvo.getFileSizeStr(size)) +#logger.infoPair("Average region file size", dvo.getFileSizeStr(size/len(files))) + +#dvo.loadRegion(329, 331.1, 1.4, 2) +#dvo.loadRegion(MINRA-BORDER, MAXRA+BORDER, MINDEC-BORDER, MAXDEC+BORDER) +#dvo.loadRegion() + +logger.infoPair("Program...", "complete") + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/removeFromDatastore.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/removeFromDatastore.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/removeFromDatastore.py (revision 33415) @@ -5,7 +5,10 @@ import sys import getopt + + +from config import Config from pslogger import PSLogger +from ipptopspsdb import IppToPspsDb -from xml.etree.ElementTree import ElementTree, Element, tostring @@ -14,12 +17,11 @@ sys.exit(1) -CONFIG = sys.argv[1] -configDoc = ElementTree(file=CONFIG) +CONFIGFILE = sys.argv[1] -logging.setLoggerClass(PSLogger) -logger = logging.getLogger("removeFromDatastore") -logger.setup(configDoc, "removeFromDatastore") - -datastore = Datastore(logger, configDoc) +# create objects +config = Config(CONFIGPATH) +logger = config.getLogger(sys.argv[0]) +ippToPspsDb = IppToPspsDb(logger, config) +datastore = Datastore(logger, config, ippToPspsDb) if len(sys.argv) < 4: Index: branches/meh_branches/ppstack_test/ippToPsps/jython/reportloadfailures.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/reportloadfailures.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/reportloadfailures.py (revision 33415) @@ -0,0 +1,50 @@ +#!/usr/bin/env jython + +from subprocess import call, PIPE, Popen +import tempfile +from xml.etree.ElementTree import ElementTree, Element, tostring +import re +import sys + +import logging.config +from pslogger import PSLogger +from ipptopspsdb import IppToPspsDb +from odm import Odm +from batch import Batch + + +if len(sys.argv) > 1: CONFIG = sys.argv[1] +else: + print "\n** Usage: " + sys.argv[0] + " \n" + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +# set up logging +logging.setLoggerClass(PSLogger) +logger = logging.getLogger(sys.argv[0]) +logger.setup(configDoc, sys.argv[0]) + +ippToPspsDb = IppToPspsDb(logger, configDoc) +odm = Odm(logger, configDoc) + +DVOLABEL = configDoc.find("dvo/gpc1Label").text +EPOCH = configDoc.find("options/epoch").text + +ids = ippToPspsDb.getFailedLoadedToODMBatchIDs(EPOCH, DVOLABEL) + + +logger.infoPair("Batches failed load to ODM", "%d" % len(ids)) + + +results = {} + +for id in ids: + + if odm.checkBatch(id, results): + logger.info(" %9s : %s" % (Batch.getNameFromID(id), results['DETAILS'])) + else: + logger.info(" %9s : failed to poll ODM" % Batch.getNameFromID(id)) + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/republishbatch.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/republishbatch.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/republishbatch.py (revision 33415) @@ -0,0 +1,60 @@ +import sys + +from pslogger import PSLogger +import logging +from xml.etree.ElementTree import ElementTree, Element, tostring + +from ipptopspsdb import IppToPspsDb +from datastore import Datastore +from batch import Batch + + +''' +A program that republishes a bath to the datastore. + +It first 'resets' the batch in the ippToPsps database, i.e. pretends it has never been published, then re-publishes it to the datastore +''' +if len(sys.argv) > 2: + CONFIG = sys.argv[1] + BATCHID = int(sys.argv[2]) +else: + print "** Usage: " + sys.argv[0] + " " + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +logging.setLoggerClass(PSLogger) +logger = logging.getLogger(sys.argv[0]) +logger.setup(configDoc, sys.argv[0]) + +ippToPspsDb = IppToPspsDb(logger, configDoc) +datastore = Datastore(logger, configDoc, ippToPspsDb) + +DVOLABEL = configDoc.find("dvo/gpc1Label").text +EPOCH = configDoc.find("options/epoch").text +BASEPATH = configDoc.find("localOutPath").text +DATASTOREPRODUCT = configDoc.find("datastore/product").text + +logger.infoTitle("Batch republisher") +logger.infoPair("Loading epoch", EPOCH) +logger.infoPair("DVO label", DVOLABEL) + + +batchName = Batch.getNameFromID(BATCHID) +BATCHTYPE = ippToPspsDb.getBatchType(BATCHID) +subDir = Batch.getSubDir(BASEPATH, BATCHTYPE, DVOLABEL) +tarballFile = Batch.getTarballFile(BATCHID) + +logger.infoPair("batch ID", "%d" % BATCHID) +logger.infoPair("batchname", batchName) +logger.infoPair("batch type", BATCHTYPE) +logger.infoPair("subdir", subDir) +logger.infoPair("tarballfile", tarballFile) +logger.infoPair("Reseting batch in database", "%d" % BATCHID) +ippToPspsDb.resetBatch(BATCHID) +logger.infoPair("publishing to", DATASTOREPRODUCT) +Batch.publishToDatastore(datastore, BATCHID, batchName, subDir, tarballFile) + + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/scratchdb.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/scratchdb.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/scratchdb.py (revision 33415) @@ -19,24 +19,70 @@ Constructor ''' - def __init__(self, logger, doc, useFull=0): - super(ScratchDb, self).__init__(logger, doc, "localdatabase") + def __init__(self, logger, config, dbName=None): + super(ScratchDb, self).__init__(logger, config, "localdatabase", dbName) + + self.dvoDoneTable = "dvoDone" + self.dvoSkyTable = "dvoSkyTable" + + + ''' + Destructor + ''' + def __del__(self): + + self.logger.debug("ScratchDb destructor") + + + ''' + Sets whether we are using normal or pre-ingested DVO tables + ''' + def setUseFullTables(self, useFull): if useFull: - self.dvoMetaTable = "dvoMetaFull" + self.dvoImagesTable = "dvoImagesFull" self.dvoDetectionTable = "dvoDetectionFull" - self.dvoDoneTable = "dvoDone" else: - self.dvoMetaTable = "dvoMeta" + self.dvoImagesTable = "dvoImages" self.dvoDetectionTable = "dvoDetection" - self.logger.debugPair("Using DVO meta table", self.dvoMetaTable) + self.logger.debugPair("Using DVO images table", self.dvoImagesTable) self.logger.debugPair("Using DVO detection table", self.dvoDetectionTable) ''' - Destructor - ''' - def __del__(self): - - self.logger.debug("ScratchDb destructor") + Creates a stored procedure for psfLikelihood. This uses an implementation of the complimentary error function taken from Numerical Recipes in C. + ''' + def createPsfLikelihoodFunction(self): + + sql = "DROP FUNCTION IF EXISTS `psfLikelihood`" + try: + self.execute(sql) + except: + self.logger.errorPair("Could not drop stored procedure", "psfLikelihood") + return False + + sql = "CREATE FUNCTION `psfLikelihood`(x REAL) RETURNS REAL \ + DETERMINISTIC \ + COMMENT 'Returns the psfLikelihood using the complimentary error function with fractional error everywhere less than 1.2 x 10-7. From Numerical Recipes in C' \ + BEGIN \ + DECLARE ans REAL; \ + DECLARE t REAL; \ + DECLARE z REAL; \ + SET x = ABS(x) / 1.4142135623731; \ + SET z=ABS(x); \ + SET t=1.0 / (1.0 + 0.5*z); \ + SET ans = t * EXP(-z*z-1.26551223+t*(1.00002368+t*(.37409196+t*(.09678418+t*(-.18628806+t*(.27886807+t*(-1.13520398+t*(1.48851587+t*(-.82215223+t*.17087277))))))))); \ + IF x < 0.0 THEN \ + SET ans = 2.0 - ans; \ + END IF; \ + RETURN ans; \ + END" + + try: + self.execute(sql) + self.logger.debugPair("Installed stored procedure", "psfLikelihood") + return True + except: + self.logger.errorPair("Could not install stored procedure", "psfLikelihood") + return False ''' @@ -56,12 +102,28 @@ return surveyID + ''' + Gets filter ID for this filter name + ''' + def getFilterID(self, name): + + sql = "SELECT filterID FROM Filter WHERE filterType = '" + name + "'" + try: + rs = self.executeQuery(sql) + rs.first() + filterID = rs.getInt(1) + except: + self.logger.errorPair("No filter ID found for filter", name) + filterID = -1 + + return filterID + ''' Gets imageID from extern ID ''' - def getImageIDFromExternID(self, sourceID, externID): + def getImageIDFromExternID(self, externID): imageID = -1 - sql = "SELECT imageID FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + sql = "SELECT IMAGE_ID FROM " + self.dvoImagesTable + " WHERE EXTERN_ID = %s" % (externID) try: rs = self.executeQuery(sql) @@ -76,9 +138,9 @@ Gets DVO image flags ''' - def getDvoImageFlags(self, sourceID, externID): + def getDvoImageFlags(self, externID): flags = 0 - sql = "SELECT flags FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + sql = "SELECT FLAGS FROM " + self.dvoImagesTable + " WHERE EXTERN_ID = %s" % (externID) try: rs = self.executeQuery(sql) @@ -92,9 +154,9 @@ Gets photcode (aka photoCalID from dvo table) ''' - def getPhotoCalID(self, sourceID, externID): + def getPhotoCalID(self, externID): photcode = -1 - sql = "SELECT photcode FROM " + self.dvoMetaTable + " WHERE sourceID = %s AND externID = %s" % (sourceID, externID) + sql = "SELECT PHOTCODE FROM " + self.dvoImagesTable + " WHERE EXTERN_ID = %s" % (externID) try: rs = self.executeQuery(sql) @@ -115,11 +177,11 @@ ''' - Inserts a new sourceID/imageID combo into dvoMeta + Inserts a new sourceID/imageID combo into dvoImages ''' def insertNewDvoExternID(self, sourceID, externID): - sql = "INSERT INTO dvoMeta ( \ - sourceID, \ - externID \ + sql = "INSERT INTO " + self.dvoImagesTable + " ( \ + SOURCE_ID, \ + EXTERN_ID \ ) VALUES (\ " + str(sourceID) + ", \ @@ -129,17 +191,11 @@ ''' - Updates dvoDone table with this DVO table - ''' - def setImportedThisDvoTable(self, name): - - sql = "INSERT INTO dvoDone (name) VALUES ('" + name + "')" - self.execute(sql) - - ''' - Have we already imported this DVO table? - ''' - def alreadyImportedThisDvoTable(self, name): - - sql = "SELECT COUNT(*) FROM dvoDone WHERE name = '" + name + "'" + Do we have an out-of-date version of this file? + ''' + def haveOutOfDateVersionOfThisDvoTable(self, path): + + fileStat = os.stat(path) + + sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate != " + str(fileStat.st_mtime) try: @@ -147,34 +203,181 @@ rs.first() if rs.getInt(1) > 0: - self.logger.errorPair("Already imported DVO tables for", name) + self.logger.debugPair("Have out-to-date version of", path) return True else: return False except: + self.logger.exception("Unable to check whether this DVO table is out-of-date") + + ''' + Have we already imported this DVO table? + Checks path and modified date to be sure + ''' + def alreadyImportedThisDvoTable(self, path): + + fileStat = os.stat(path) + + sql = "SELECT COUNT(*) FROM " + self.dvoDoneTable + " WHERE path = '" + path + "' AND modifiedDate = " + str(fileStat.st_mtime) + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) > 0: + self.logger.debugPair("Already imported up-to-date version of", path) + return True + else: + return False + except: self.logger.exception("Unable to check whether this DVO table has been imported") ''' + Updates dvoDone table with this DVO table + ''' + def setImportedThisDvoTable(self, path, region="null"): + + fileStat = os.stat(path) + + # first delete any old version we have ingested + sql = "DELETE FROM " + self.dvoDoneTable + " WHERE path = '" + path + "'" + self.execute(sql) + + # now insert new version with up-to-date size and date + sql = "INSERT INTO " + self.dvoDoneTable + " (region, path, modifiedDate, size) VALUES ('" + region + "', '" + path + "', " + str(str(fileStat.st_mtime)) + ", " + str(str(fileStat.st_size)) + ")" + self.execute(sql) + + + ''' + Gets the id for the cpm file for the provided region + ''' + def getDvoCpmFileIdForRegion(self, region): + + id = -1 + sql = "SELECT id FROM " + self.dvoDoneTable + " WHERE region = '" + region + "' AND path LIKE '%.cpm'"; + try: + rs = self.executeQuery(sql) + rs.first() + id = rs.getInt(1) + except: + self.logger.errorPair("Could not get file id from " + self.dvoDoneTable, "for region: " + region) + + return id + + + ''' + Gets the id for the provided file + ''' + def getDvoFileId(self, path): + + id = -1 + sql = "SELECT id FROM " + self.dvoDoneTable + " WHERE path = '" + path + "'" + try: + rs = self.executeQuery(sql) + rs.first() + id = rs.getInt(1) + except: + self.logger.errprPair("Could not get file id from " + self.dvoDoneTable + "for", path) + + return id + + ''' + Returns a list of DVO regions currently ingested + ''' + def getIngestedDvoRegions(self): + + regions = [] + + sql = "SELECT DISTINCT region FROM dvoDone WHERE region != 'null'" + try: + rs = self.executeQuery(sql) + while (rs.next()): + regions.append(rs.getString(1)) + rs.close() + except: + self.logger.errorPai("Can't query for ingested regions", sql) + + return regions + + + ''' + Gets total size of DVO files imported to this database + Counts only cpm and cpt files + ''' + def getTotalSizeOfIngestedDvoFiles(self): + + size = 0 + sql = "SELECT SUM(size) FROM " + self.dvoDoneTable + " WHERE path LIKE '%cpm' OR path LIKE '%cpt'" + try: + rs = self.executeQuery(sql) + rs.first() + size = rs.getLong(1) + except: + self.logger.errorPair("Unable to get total ingested DVO size", sql) + + return size + + ''' + Checks whether DVO files we have ingested are form the DVO path provided + ''' + def isCorrectDvo(self, path): + + sql = "SELECT path FROM " + self.dvoDoneTable + " where path LIKE '%Images.dat'" + try: + rs = self.executeQuery(sql) + rs.first() + + head,tail = os.path.split(rs.getString(1)) + + if head == path: + self.logger.debugPair("Already ingested stuff from DVO at", path) + return True + else: + self.logger.errorPair("DVO does not match. We have", head) + return False + + except: + self.logger.errorPair("No DVO files ingested for this path", path) + return False + + ''' + Checks whether the astrometric solution is ok for this chip + TODO the value of 50 for numAstroRef is hardcoded here, but this should be temporary anyway + ''' + def astrometricSolutionOK(self, ota): + + sql = "SELECT numAstroRef FROM ImageMeta_" + ota + + try: + rs = self.executeQuery(sql) + rs.first() + if rs.getInt(1) < 50: + self.logger.debug("Bad astrometric solution for", ota) + return False + else: + return True + except: + self.logger.debug("Unable to check astrometric solution") + + return True + + + ''' Creates a table for for ID matching ''' def createDvoTables(self): - - self.logger.debugPair("Creating DVO table", "dvoMeta") - - sql = "DROP TABLE dvoMeta" + + # drop and create Images table + self.logger.debugPair("Creating DVO table", self.dvoImagesTable) + sql = "DROP TABLE " + self.dvoImagesTable try: self.execute(sql) except: pass - sql = "DROP TABLE dvoDetection" - try: self.execute(sql) - except: pass - - sql = "CREATE TABLE dvoMeta ( \ - sourceID INT, \ - imageID INT, \ - externID INT, \ - flags INT, \ - photcode INT, \ - PRIMARY KEY (sourceID, imageID, externID) \ + sql = "CREATE TABLE " + self.dvoImagesTable + " ( \ + SOURCE_ID SMALLINT, \ + IMAGE_ID INT, \ + EXTERN_ID INT, \ + FLAGS INT, \ + PHOTCODE SMALLINT, \ + PRIMARY KEY (IMAGE_ID, EXTERN_ID) \ )" @@ -183,7 +386,12 @@ self.logger.error("Unable to create DVO meta-data database table") + # now detection table + self.logger.debugPair("Creating DVO table", self.dvoDetectionTable) + sql = "DROP TABLE " + self.dvoDetectionTable + try: self.execute(sql) + except: pass + self.logger.debugPair("Creating DVO table", "dvoDetection") sql = "CREATE TABLE dvoDetection ( \ - sourceID INT, \ imageID INT, \ ippDetectID BIGINT, \ @@ -192,5 +400,5 @@ objID BIGINT, \ flags INT, \ - PRIMARY KEY (sourceID, imageID, ippDetectID) \ + PRIMARY KEY (imageID, ippDetectID) \ )" @@ -201,30 +409,119 @@ ''' - Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this... - ''' - def resetDvoToMysqlTables(self): - - self.logger.infoPair("Dropping table", self.dvoMetaTable) - self.dropTable(self.dvoMetaTable) + Removes the provided DVO FITS files from from dvoDone and dvoDetection tables + Returns a count of the number of detections deleted + ''' + def purgeTheseDvoRegions(self, regions): + + detectionCount = self.getRowCount(self.dvoDetectionTable) + for region in regions: + + fileId = self.getDvoCpmFileIdForRegion(region) + + if fileId < 0: + self.logger.errorPair("Unable to get file ID to delete " + region + " from", self.dvoDoneTable) + continue + + sql = "DELETE FROM " + self.dvoDetectionTable + " WHERE fileID = " + str(fileId) + try: self.execute(sql) + except: + self.logger.errorPair("Unable to delete from " + self.dvoDetectionTable, "for file ID: %d" % fileId) + return False + + sql = "DELETE FROM " + self.dvoDoneTable + " WHERE region = '" + region + "'" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to delete " + region + " from", self.dvoDoneTable) + + finalCount = detectionCount - self.getRowCount(self.dvoDetectionTable) + + # after a big delete, we should OPTIMIZE the table + self.logger.infoPair("Running OPTIMIZE on ", self.dvoDetectionTable) + if finalCount > 0: self.optimizeTable(self.dvoDetectionTable) + + return finalCount + + ''' + Drops DVO stuff pertaining to detections + ''' + def resetDvoDetectionTables(self): + + self.logger.infoPair("Deleting detections from", self.dvoDoneTable) + sql = "DELETE FROM " + self.dvoDoneTable + " WHERE path LIKE '%cpm' OR path LIKE '%cpt'" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to delete from", self.dvoDoneTable) + return False + self.logger.infoPair("Dropping table", self.dvoDetectionTable) self.dropTable(self.dvoDetectionTable) - self.logger.infoPair("Dropping table", self.dvoDoneTable) - self.dropTable(self.dvoDoneTable) - - self.logger.infoPair("Creating table", self.dvoMetaTable) - sql = "CREATE TABLE " + self.dvoMetaTable + " LIKE dvoMeta" - try: self.execute(sql) - except: self.logger.errorPair("Unable to create table", self.dvoMetaTable) self.logger.infoPair("Creating table", self.dvoDetectionTable) sql = "CREATE TABLE " + self.dvoDetectionTable + " LIKE dvoDetection" try: self.execute(sql) - except: self.logger.errorPair("Unable to create table", self.dvoDetectionTable) - + except: + self.logger.errorPair("Unable to create table", self.dvoDetectionTable) + return False + + # TODO + sql = "ALTER TABLE " + self.dvoDetectionTable + " ADD fileID INT NOT NULL" + try: self.execute(sql) + except: + self.logger.errorPair("Unable to add fileID column to ", self.dvoDetectionTable) + return False + + ''' + Drops and recreates tables necessary for dvoToMySQL program. Be very careful before using this... + ''' + def resetAllDvoTables(self): + + self.logger.infoPair("Dropping table", self.dvoImagesTable) + self.dropTable(self.dvoImagesTable) + self.logger.infoPair("Dropping table", self.dvoSkyTable) + self.dropTable(self.dvoSkyTable) + self.logger.infoPair("Dropping table", self.dvoDoneTable) + self.dropTable(self.dvoDoneTable) + + # create dvoDone table self.logger.infoPair("Creating table", self.dvoDoneTable ) - sql = "CREATE TABLE " + self.dvoDoneTable + " (name VARCHAR(100))" + sql = "CREATE TABLE " + self.dvoDoneTable + " (id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, region VARCHAR(100), path VARCHAR(1000), modifiedDate BIGINT, size BIGINT)" try: self.execute(sql) except: self.logger.errorPair("Unable to create table", self.dvoDoneTable) - - + self.resetDvoDetectionTables() + + ''' + Gets a list of DVO regions that overlap with the defined box + ''' + def getDvoRegions(self, minRa, maxRa, minDec, maxDec): + + self.logger.debug("Querying DVO SkyTable for FITS files in this region") + + regions = [] + self.getDvoRegionsForThisBox(regions, minRa, maxRa, minDec, maxDec) + + # deal with 0/360 boundary + if minRa < 0: self.getDvoRegionsForThisBox(regions, minRa + 360, 361 ,minDec, maxDec) + if maxRa > 360: self.getDvoRegionsForThisBox(regions, 0, maxRa - 360 ,minDec, maxDec) + + return regions + + ''' + Gets a list of DVO regions that overlap with the defined box + ''' + def getDvoRegionsForThisBox(self, regions, minRa, maxRa, minDec, maxDec): + + sql = "SELECT name FROM " + self.dvoSkyTable + " \ + WHERE NOT ((" + str(maxRa) + " <= R_MIN) OR \ + (" + str(minRa) + " >= R_MAX) OR \ + (" + str(maxDec) + " <= D_MIN) OR \ + (" + str(minDec) + " >= D_MAX)) " + + try: + rs = self.executeQuery(sql) + while (rs.next()): regions.append(rs.getString(1)) + rs.close() + except: + self.logger.exception("Can't query for DVO regions") + + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/setupScratchDb.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/setupScratchDb.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/setupScratchDb.py (revision 33415) @@ -0,0 +1,49 @@ +#!/usr/bin/env jython + +import os.path +import sys +import time +import stilts +from java.lang import * +from java.sql import * +import logging.config + +from java.lang import Math +from org.apache.commons.math.special import Erf +from xml.etree.ElementTree import ElementTree, Element, tostring + +from pslogger import PSLogger +from scratchdb import ScratchDb + +''' +Start of program. +''' + +if len(sys.argv) > 1: CONFIG = sys.argv[1] +else: + print "** Usage: " + sys.argv[0] + " [init]" + sys.exit(1) + +# open config file +configDoc = ElementTree(file=CONFIG) + +# set up logging +logging.setLoggerClass(PSLogger) +logger = logging.getLogger("setupScratchDb") +logger.setup(configDoc, "setupScratchDb") + +scratchDb = ScratchDb(logger, configDoc) + +# install stored procedure +logger.info("Installing psfLikelihood() stored procedure") +scratchDb.createPsfLikelihoodFunction() + +# install IN tables +logger.info("Installing initialization tables") +tables = stilts.treads("../config/IN/tables.vot") +for table in tables: + logger.debug("Creating IN table: " + table.name) + table.write(scratchDb.url + '#' + table.name) + +logger.info("Done") + Index: branches/meh_branches/ppstack_test/ippToPsps/jython/stackbatch.py =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/jython/stackbatch.py (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/jython/stackbatch.py (revision 33415) @@ -19,4 +19,7 @@ ''' StackBatch class + +This class, a sub-class of Batch, processes single stack skycells in the form of IPP cmf files + ''' class StackBatch(Batch): @@ -27,19 +30,23 @@ def __init__(self, logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, - stackID): + scratchDb, + stackID, + batchID, + useFullTables): super(StackBatch, self).__init__( logger, - configPath, - configDoc, + config, gpc1Db, ippToPspsDb, + scratchDb, stackID, + batchID, "ST", - gpc1Db.getStackStageCmf(stackID)) + gpc1Db.getStackStageCmf(config.dvoLabel, stackID), + useFullTables) if not self.everythingOK: return @@ -49,11 +56,26 @@ # get stack meta data meta = self.gpc1Db.getStackStageMeta(self.id) + if not meta: + self.everythingOK = False + return + self.filter = meta[0]; self.filter = self.filter[0:1] + self.filterID = self.scratchDb.getFilterID(self.filter) self.skycell = meta[1]; - # TODO HACK fix this + + # skycell is, eg "skycell.1133.081" + # we need 1133.081 for skycell, but store as an int, so instead store 1133081 + # projection cell is just the 1133 bit + # this is how it should be done + #self.projectioncell = self.skycell[8:12] + #self.skycell = self.skycell[8:] + #self.skycell = self.skycell.replace(".", "") + #self.projectioncell = self.skycell[8:12] + + # TODO this is the wrong way, but is consistent with SkyCell table in IN data self.skycell = self.skycell[8:12] - # mangling e.g. 0683.043 into 0683043 for now until we have a schema change - #self.skycell = self.skycell.replace(".", "") + self.projectioncell = self.skycell + self.analysisVer = meta[2]; @@ -77,5 +99,5 @@ self.scratchDb.dropTable("SkyChip_xsrc") - # create an output filename, which is {filterID}{skycellID}.FITS + # create an output filename, which is {stackID}.FITS self.outputFitsFile = "%08d.FITS" % self.id self.outputFitsPath = "%s/%s" % (self.localOutPath, self.outputFitsFile) @@ -90,5 +112,4 @@ if not self.useFullTables: self.scratchDb.insertNewDvoExternID(self.header['SOURCEID'], self.header['IMAGEID']) - # dump stuff to log @@ -96,7 +117,6 @@ self.logger.infoPair("Stack type", "%s" % self.stackType) self.logger.infoPair("Skycell", "%s" % self.skycell) + self.logger.infoPair("Projectioncell", "%s" % self.projectioncell) self.logger.infoPair("Filter", "%s" % self.filter) - - ''' @@ -247,4 +267,6 @@ ,expTime \ ,psfModelID \ + ,psfFwhm_mean \ + ,psfFwhm_max \ ,ctype1 \ ,ctype2 \ @@ -266,4 +288,6 @@ ," + str(self.expTime) + " \ ,'" + self.safeDictionaryAccess(self.header, 'PSFMODEL') + "' \ + ,'" + self.safeDictionaryAccess(self.header, 'FWHM_MAJ') + "' \ + ,'" + self.safeDictionaryAccess(self.header, 'FW_MJ_UQ') + "' \ ,'" + self.header['CTYPE1'] + "' \ ,'" + self.header['CTYPE2'] + "' \ @@ -283,5 +307,5 @@ self.scratchDb.updateAllRows("StackMeta", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackMeta", self.filter) - self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackMeta", "dataRelease", str(self.config.dataRelease)) self.updateStackTypeID("StackMeta") @@ -297,4 +321,5 @@ ippDetectID \ ,skyCellID \ + ,projectionCellID \ ,obsTime \ ,xPos \ @@ -311,4 +336,5 @@ ,psfWidMinor \ ,psfTheta \ + ,psfLikelihood \ ,infoFlag \ ,psfCf \ @@ -335,4 +361,5 @@ IPP_IDET \ ," + self.skycell + " \ + ," + self.projectioncell + " \ ," + self.header['MJD-OBS'] + " \ ,X_PSF \ @@ -349,4 +376,5 @@ ,PSF_MINOR \ ,PSF_THETA \ + ,psfLikelihood(EXT_NSIGMA) \ ,FLAGS << 32 | FLAGS2 \ ,PSF_QF \ @@ -375,5 +403,5 @@ self.scratchDb.updateAllRows("StackDetection", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackDetection", self.filter) - self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackDetection", "dataRelease", str(self.config.dataRelease)) self.scratchDb.updateAllRows("StackDetection", "primaryF", "0") self.scratchDb.updateAllRows("StackDetection", "activeFlag", "0") @@ -381,7 +409,7 @@ self.updateStackTypeID("StackDetection") self.updateDvoIDs("StackDetection") - sql = "ALTER TABLE StackDetection ADD PRIMARY KEY (objID, stackDetectID)" - self.scratchDb.execute(sql) - + sql = "ALTER IGNORE TABLE StackDetection ADD PRIMARY KEY (objID)" + self.scratchDb.execute(sql) + if self.stackType == "DEEP_STACK": @@ -390,7 +418,7 @@ SET instFlux = 2*b.PSF_INST_FLUX_SIG \ WHERE instFlux IS NULL \ + AND a.ippDetectID = b.IPP_IDET \ AND b.PSF_INST_FLUX_SIG IS NOT NULL" self.scratchDb.execute(sql) - # instFlux = 2*PSF_INST_FLUX_SIG self.scratchDb.reportAndDeleteRowsWithNULLS("StackDetection", "instFlux") @@ -417,5 +445,5 @@ # TODO temporarily loading 1st convolved fluxes into unconvolved fields self.logger.infoPair("Adding fluxes", "un-convolved") - self.updateApFlxs("", "< 7.0") + self.updateApFlxs("", "IS NULL") self.logger.infoPair("Adding fluxes", "1st convolved") self.updateApFlxs("c1", "< 7.0") @@ -433,4 +461,5 @@ ,petR90=b.PETRO_RADIUS_90 \ ,petR90Err=b.PETRO_RADIUS_90_ERR \ + ,petCf=b.PETRO_FILL \ WHERE a.ippDetectID=b.IPP_IDET" self.scratchDb.execute(sql) @@ -438,5 +467,5 @@ self.scratchDb.updateAllRows("StackApFlx", "surveyID", str(self.surveyID)) self.scratchDb.updateFilterID("StackApFlx", self.filter) - self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.dataRelease)) + self.scratchDb.updateAllRows("StackApFlx", "dataRelease", str(self.config.dataRelease)) self.scratchDb.updateAllRows("StackApFlx", "primaryF", "0") self.scratchDb.updateAllRows("StackApFlx", "activeFlag", "0") @@ -490,5 +519,4 @@ rs.first() nMissing = rs.getInt(1) - self.logger.infoPair("Detections deleted from %s that are not in StackDetection" + table, "%5d" % nMissing) if nMissing < 1: return @@ -496,4 +524,5 @@ sql = "DELETE FROM " + table + " WHERE ippDetectID NOT IN (SELECT ippDetectID FROM StackDetection)" self.scratchDb.execute(sql) + self.logger.infoPair("Detections not in StackDetection", "%d deleted from %s" % (nMissing, table)) @@ -527,8 +556,10 @@ objID \ ,ippObjID \ + ,projectionCellID \ ) \ SELECT \ DISTINCT objID \ ,ippObjID \ + ,projectionCellID \ FROM StackDetection" self.scratchDb.execute(sql) @@ -569,4 +600,6 @@ self.scratchDb.createIndex("StackModelFit", "ippDetectID") + return True + ''' Applies indexes to the IPP tables @@ -580,4 +613,6 @@ self.scratchDb.createIndex("SkyChip_xsrc", "IPP_IDET") + return True + ''' Updates provided table with DVO IDs from DVO table @@ -587,7 +622,7 @@ imageID = self.scratchDb.getImageIDFromExternID(self.header['SOURCEID'], self.header['IMAGEID']) self.logger.debug("Updating table '" + table + "' with DVO IDs...") - sql = "UPDATE IGNORE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ + sql = "UPDATE " + table + " AS a, " + self.scratchDb.dvoDetectionTable + " AS b SET \ a.ippObjID = b.ippObjID, \ - a.stackDetectID = b.detectID, \ + a.stackDetectID = 10 * b.detectID + " + str(self.filterID) + ", \ a.objID = b.objID \ WHERE a.ippDetectID = b.ippDetectID \ Index: branches/meh_branches/ppstack_test/ippToPsps/perl/checkOdmStatus.pl =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/perl/checkOdmStatus.pl (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/perl/checkOdmStatus.pl (revision 33415) @@ -22,9 +22,4 @@ $ua->env_proxy; -my $loadedToOdm; -my $loadFailed; -my $mergeWorthy; -my $merged; - my $statusFilter = "*"; @@ -42,56 +37,13 @@ exit(1); } -# print( "HTTP response status: ".$response->content."\n" ); open (DAT, ">".$file); -#my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_odmXml.XXXX", UNLINK => 0); print DAT $response->content; -#print $response->content; -#close($tempFile); -#${$loadedToOdm} = 0; -#${$loadFailed} = 0; -#${$mergeWorthy} = 0; -#${$merged} = 0; print "$file\n"; -#parseXml($tempName, $loadedToOdm, $loadFailed, $mergeWorthy, $merged); exit(0); - #unlink($tempFile); -######################################################################################## -# -# Parses ODM XML -# -######################################################################################## - sub parseXml { - my ($xmlFile, $loadedToOdm, $loadFailed, $mergeWorthy, $merged) = @_; - - my $parser = XML::LibXML->new; - my $doc = $parser->parse_file($xmlFile); - my $xc = XML::LibXML::XPathContext->new($doc); - $xc->registerNs('ArrayOfOdmBatchState', 'PanSTARRS.Services.OdmWebService'); - my $result = $xc->findvalue('//ArrayOfOdmBatchState:BatchState'); - - if ($result =~ m/failed/) { - ${$loadFailed} = 1; - ${$loadedToOdm} = 1; - return; - } - - $result = $xc->findvalue('//ArrayOfOdmBatchState:Message'); - ${$loadedToOdm} = 0; - ${$mergeWorthy} = 0; - ${$merged} = 0; - - if ($result =~ m/None/) { ${$loadedToOdm} = 1;} - if ($result =~ m/LoadStarted/) { ${$loadedToOdm} = 1;} - if ($result =~ m/MergeWorthy/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1;} - if ($result =~ m/Merge[1-9]Completed/) { ${$loadedToOdm} = 1; ${$mergeWorthy} = 1; ${$merged} = 1;} - - - - } Index: branches/meh_branches/ppstack_test/ippToPsps/perl/deleteFromDXLayer.pl =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/perl/deleteFromDXLayer.pl (revision 33415) +++ branches/meh_branches/ppstack_test/ippToPsps/perl/deleteFromDXLayer.pl (revision 33415) @@ -0,0 +1,61 @@ +#!/usr/bin/env perl + +use warnings; +use strict; + +use LWP::UserAgent; +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use XML::LibXML; +use File::Temp qw(tempfile); + + +my $batchName = undef; +my $file = undef; + +GetOptions( + 'batch|b=s' => \$batchName, + 'file|f=s' => \$file, + ); + +my $dxWebServiceUrl = "http://lsb01.psps.ifa.hawaii.edu/cgi-bin/dxlayer/dxwebservice.cgi"; +#my $dxWebServiceUrl = "http://166.122.172.121/cgi-bin/dxlayer/dxwebservice.cgi"; +my $ua = LWP::UserAgent->new; +$ua->timeout(15); +$ua->env_proxy; + + my $doc = XML::LibXML::Document->new(); # Global document variable. + my $rootNode = $doc->createElement('AdminQueryParams'); +$doc->setDocumentElement($rootNode); + +my $paramNode = $doc->createElement('Param'); +$paramNode->setAttribute('Name', "BatchID"); # default FirstRow is 1 +my $text=XML::LibXML::Text->new($batchName); +$paramNode->appendChild($text); #append +$rootNode->appendChild($paramNode); # append a node to the xml tree + +my $xmlParams = $doc->toString; + +print $xmlParams; + +my $response = $ua->post($dxWebServiceUrl, + [op => "delete_dxbatch_files", + xmlParams => $xmlParams ] + ); + +# '200' is the 'OK' response +if ($response->code != 200) { + + print "Problem connecting to DXLayerweb service, HTTP response status: ".$response->status_line."\n"; + exit(1); +} + +open (DAT, ">".$file); +print DAT $response->content; + + +print "$file\n"; + +exit(0); + + + Index: branches/meh_branches/ppstack_test/ippToPsps/perl/exposureSummary.pl =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/perl/exposureSummary.pl (revision 32161) +++ (revision ) @@ -1,236 +1,0 @@ -#!/usr/bin/env perl - -# script to provide a summary of loading status given an upper and lower exposure ID limit - -use warnings; -use strict; - -use LWP::UserAgent; -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use File::Temp qw(tempfile); -use ippToPsps::IppToPspsDb; -use ippToPsps::Gpc1Db; -use ippToPsps::DetectionBatch; - -my $dvoDb = undef; -my $beginExp = undef; -my $endExp = undef; -my $verbose = undef; -my $checkNeb = undef; -my $save_temps = undef; - -GetOptions( - 'startexp|b=s' => \$beginExp, - 'endexpi|e=s' => \$endExp, - 'dvodb|d=s' => \$dvoDb, - 'checkNeb|n' => \$checkNeb, - 'verbose|v' => \$verbose, - 'save_temps|s' => \$save_temps - ); - - -my $quit = 0; -print "\n*******************************************************************************\n"; -print "* \n"; -if (@ARGV) { - $quit=1; - print "* UNKNKOWN: option @ARGV\n"; -} -if (!defined $beginExp) { - $quit = 1; - print "* REQUIRED: a begin exp_id -b \n"; -} -if (!defined $endExp) { - $quit = 1; - print "* REQUIRED: an end exp_id -e \n"; -} -if (!defined $dvoDb) { - $quit = 1; - print "* REQUIRED: a dvo Db name -d \n"; -} -if (!defined $checkNeb) { - $checkNeb = 0; - print "* OPTIONAL: check nebulous for unprocessed exposures -n (default = $checkNeb)\n"; -} -if (!defined $verbose) { - $verbose = 0; - print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; -} -if (!defined $save_temps) { - $save_temps = 0; - print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; -} -print "*\n*******************************************************************************\n"; - -if ($quit) { exit; } - -my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps); -my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); - -my $exposures; -#if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures, 0)) {exit;} - -my $totalExposures = $ippToPspsDb->countExposures($beginExp, $endExp, "PSPS_test", "P2", $dvoDb); -my $totalMerged = $ippToPspsDb->countMergedExposures($beginExp, $endExp, "PSPS_test", "P2", $dvoDb); - -if(!$ippToPspsDb->getUnmergedExposures(\$exposures, $beginExp, $endExp, "PSPS_test", "P2", $dvoDb)) { - - print "No unmerged exposures\n"; - exit; -} - -my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); -my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; - -my $exposure; -my $totalUnmerged = 0; -my $totalMergeWorthy = 0; -my $totalNotMergeWorthy = 0; -my $totalProcessed = 0; -my $totalOnDatastore = 0; -my $totalLoadedToOdm = 0; -my $totalLoadFailed = 0; -my $totalMergeWorthy = 0; -my $totalUnderObjIdLimit = 0; -my $totalUnprocessed = 0; -my $totalUnprocessedAvailableSmfs = 0; -my $remainingOdmFailures = 0; -my $totalUnprocessedMissingSmfs = 0; -my $totalProcessedNotOnDatastore = 0; - - -open (UNPRO, ">unprocessedExposures.txt") or print "* Problem opening data file for unprocessed exposures\n"; -open (ODMFAIL, ">odmFailures.txt") or print "* Problem opening data file for ODM failures\n"; -open (NOTONDATASTORE, ">notOnDatastore.txt") or print "* Problem opening data file for exposures not on datastore\n"; - -if ($checkNeb) { - - open (MISSSMF, ">missingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; - open (NOTMISSSMF, ">notMissingSmfs.txt") or print "* Problem opening data file for missing smfs\n"; -} - -foreach $exposure ( @{$exposures} ) { - my ($batchId, $expId) = @{$exposure}; - - if ($ippToPspsDb->isExposureMerged($expId)) {next;} - - $totalUnmerged++; - if ($verbose) {print "$totalUnmerged | $expId \n"; } - - # processed stuff - if ($ippToPspsDb->isExposureProcessed($expId)) { - - $totalProcessed++; - - if (!$ippToPspsDb->isExposureAlreadyPublished($expId)) { - - $totalProcessedNotOnDatastore++; - print NOTONDATASTORE "$expId\n"; - } - else { - - $totalOnDatastore++; - - if ($ippToPspsDb->isExposureLoadedToOdm($expId)) { - - $totalLoadedToOdm++; - - if ($ippToPspsDb->isMergeWorthy($expId)) { - - $totalMergeWorthy++; - } - # not mergeworthy - else { - - $totalNotMergeWorthy++; - - #if ($ippToPspsDb->didLoadFail($expId)) { - - # $totalLoadFailed++; - #} - if ($ippToPspsDb->isMinObjIdUnderLimit($expId)) { - - $totalUnderObjIdLimit++; - } - else { - - $remainingOdmFailures++; - print ODMFAIL "$expId\n"; - } - } - } - } - } - # unprocessed stuff - else { - - - $totalUnprocessed++; - print UNPRO "$expId\n"; - - # if flag is set, then attempt to get smf file from nebulous for this unprocessed exposure - if ($checkNeb) { - - my $detectionBatch = ippToPsps::DetectionBatch->existing( - "GPC1", - $gpc1Db, - $batchId, - $ippToPspsDb, - ".", - $verbose, - $save_temps); - - - my $path = $detectionBatch->getSmfFile(); - if (!$path) { - print "no smf file for $expId\n"; - - $totalUnprocessedMissingSmfs++; - print MISSSMF "$expId\n"; - } - else { - - $totalUnprocessedAvailableSmfs++; - print NOTMISSSMF "$expId\n"; - - } - } - - } -} - -close(UNPRO); -close(ODMFAIL); -close(NOTONDATASTORE); -if ($checkNeb) { - - close(MISSSMF); - close(NOTMISSSMF); -} -my $totalOnDatastoreNotLoaded = $totalOnDatastore - $totalLoadedToOdm; -print "***********************************************************\n"; -print "* Summary of IPP->PSPS loading as of $timeStamp\n"; -print "* DVO Db = '$dvoDb'\n"; -print "* Exposures between $beginExp and $endExp\n"; -print "*\n"; -print "* Total Exposures = $totalExposures\n"; -print "* Merged into PSPS = $totalMerged\n"; -print "* Un-merged = $totalUnmerged\n"; -print "* Unprocessed by ippToPsps = $totalUnprocessed\n"; -if ($checkNeb) { - print "* Missing SMF files = $totalUnprocessedMissingSmfs\n"; - print "* Available SMF files = $totalUnprocessedAvailableSmfs\n"; -} -print "* Processed by ippToPsps = $totalProcessed\n"; -print "* Not on datastore = $totalProcessedNotOnDatastore\n"; -print "* On datastore = $totalOnDatastore\n"; -print "* Not loaded to ODM = $totalOnDatastoreNotLoaded\n"; -print "* Loaded to ODM = $totalLoadedToOdm\n"; -print "* MergeWorthy = $totalMergeWorthy\n"; -print "* Not MergeWorthy = $totalNotMergeWorthy\n"; -print "* Under -30 dec limit = $totalUnderObjIdLimit\n"; -print "* Remaining failures = $remainingOdmFailures\n"; -print "*\n"; -print "***********************************************************\n"; - - Index: branches/meh_branches/ppstack_test/ippToPsps/perl/ippToPsps_run.pl =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/perl/ippToPsps_run.pl (revision 32161) +++ (revision ) @@ -1,554 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; - -use PS::IPP::Config 1.01 qw( :standard ); -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use IPC::Cmd 0.36 qw( can_run run ); -use File::Temp qw(tempfile); -use XML::LibXML; -use File::Basename; - -# local classes -use ippToPsps::Gpc1Db; -use ippToPsps::IppToPspsDb; -use ippToPsps::Datastore; -#use ippToPsps::BatchManager; - -# globals -my $camera = undef; -my $batchType = undef; -my $dvoLocation = undef; -my $dvoDb = undef; -my $fullDvoPath = undef; -my $verbose = undef; -my $save_temps = undef; -my $output = undef; -my $singleExpId = undef; -my $datastoreProduct = undef; -my $force = undef; -my $initBatch = undef; -my $dontTarNZip = undef; - -# get user args -GetOptions( - 'camera|c' => \$camera, - 'output|o=s' => \$output, - 'batch|b=s' => \$batchType, - 'dvo|d=s' => \$fullDvoPath, - 'expid|e=s' => \$singleExpId, - 'product|p=s' => \$datastoreProduct, - 'verbose|v' => \$verbose, - 'save_temps|t' => \$save_temps, - 'force|f' => \$force, - 'tarnzip|z' => \$dontTarNZip, - ); - -my $quit = 0; -print "\n*******************************************************************************\n"; -print "* \n"; -if (@ARGV) { - $quit=1; - print "* UNKNKOWN: option @ARGV\n"; -} -if (!defined $output) { - $quit=1; - print "* REQUIRED: need to provide an output path -o \n"; -} -if (!defined $batchType) { - $quit=1; - print "* REQUIRED: need to define a batch type -b \n"; -} -if (!defined $fullDvoPath) { - $quit=1; - print "* REQUIRED: need to provide a DVO Db -d \n"; -} -if (!defined $camera) { - $camera = "GPC1"; - print "* OPTIONAL: select a camera -c (default = $camera)\n"; -} -if (!defined $singleExpId) { - - print "* OPTIONAL: a single exposure ID -e (default = none)\n"; -} -if (!defined $datastoreProduct) { - - print "* OPTIONAL: datastore product -p (default = none, i.e. data will not be published)\n"; -} -if (!defined $verbose) { - $verbose = 0; - print "* OPTIONAL: run in verbose mode -v (default = $verbose)\n"; -} -if (!defined $save_temps) { - $save_temps = 0; - print "* OPTIONAL: keep temp files -t (default = $save_temps)\n"; -} -if (!defined $force) { - $force = 0; - print "* OPTIONAL: force if already processed -f (default = $force)\n"; -} -if (!defined $dontTarNZip) { - $dontTarNZip = 0; - print "* OPTIONAL: don't tar and zip output -z (default = $dontTarNZip)\n"; -} -print "*\n*******************************************************************************\n"; - -if ($quit) { exit; } - -# determine PSPS batch 'type' -my $pspsBatchType; -if ($batchType eq 'init') {$pspsBatchType = "IN";} -elsif ($batchType eq 'det') {$pspsBatchType = "P2";} -elsif ($batchType eq 'stack') {$pspsBatchType = "ST";} -elsif ($batchType eq 'diff') {$pspsBatchType = "OB";} -else {$pspsBatchType = "UNKNOWN";} - -# spilt full DVO path into Db and path -$fullDvoPath =~ s/\/$//; # strip off trailing '\' -($dvoDb, $dvoLocation) = fileparse($fullDvoPath); - -my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps); -my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps); -my $datastore = undef; -if ($datastoreProduct) {$datastore = new ippToPsps::Datastore($datastoreProduct, $verbose, $save_temps);} -#my $batchManager = new ippToPsps::BatchManager($ippToPspsDb, $output, $verbose, $save_temps); - -# check we can run programs and get camera config -my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR)); -my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); - -if ($batchType eq "init") {$initBatch = 1;} -else {$initBatch = 0;} - -if (!process()) {print "* Finished unsuccessfully\n";} -else {print "* Finished successfully\n";} - -####################################################################################### -# -# Generates a PSPS-suitable survey 'type' from the IPP distribution group -# -####################################################################################### -sub getSurveyTypeFromDistGroup { - my ($distGroup) = @_; - - if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";} - if ($distGroup =~ m/^M31$/i) {return "M31";} - if ($distGroup =~ m/^sts$/i) {return "STS1";} - if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";} - if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";} - - print "* Unknown distribution group: '$distGroup'\n"; - return undef; -} - -####################################################################################### -# -# Finds all exposures for the provided TODO and generates PSPS FITS data for them -# -####################################################################################### -sub process { - - my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps); - my $lastExpId = $ippToPspsDb->getLastExpId(); - my $exposures; - - my $query; - if ($initBatch) { - - my @exposure = (0, 'NULL', 'ThreePi'); - @{$exposures} = (\@exposure); - } - # get single exposure - elsif ($singleExpId) { - - if (!$gpc1Db->getSingleExposureFromDvoDb($dvoDb, $singleExpId, \$exposures)) {return 0;} - } - # get all exposures in this DVO Db - else { - - if (!$gpc1Db->getExposureListFromDvoDb($dvoDb, \$exposures)) {return 0;} - } - - my $batchId = 0; - my $batchesPerJob = 1; - my $published = 0; - - #my $batchId = -1; - my $exposure; - foreach $exposure ( @{$exposures} ) { - my ($expId, $expName, $distGroup) = @{$exposure}; - - if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) { - - if ($force) {print "* Already processed '$expId', but forcing....\n";} - else { - - print "* Exposure ID '$expId' has already been processed\n"; - next - }; - } - - my $surveyType = getSurveyTypeFromDistGroup($distGroup); - if (!$surveyType) {next;} - - $batchId = $ippToPspsDb->createNewBatch($expId, $distGroup, $pspsBatchType, $dvoDb, (defined $datastore) ? $datastoreProduct : "NONE"); - - # TODO quit here if no sensible batch ID - - # generate batch path from batch IDs - my $batch = sprintf("B%08d", $batchId); - my $dvoDir = "$output/$dvoDb"; - my $batchDir = "$dvoDir/$batch"; - - # make directories - unless(-d $dvoDir) {mkdir($dvoDir, 0777);} - mkdir($batchDir, 0777); - - $published = 0; - - print "* Preparing exposure $expId as batch '$batch'...\n"; - - # run IppToPsps program - if (runIppToPsps($expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath)) { - - my ($filename, $minObjId, $maxObjId, $totalDetections); - parseResults($resultsFilePath, \$filename, \$minObjId, \$maxObjId, \$totalDetections); - - # write batch manifest and tar and zip up - if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) { - - # tar n' zip - if (!$dontTarNZip) { - my $tarball = tarAndZipBatch($dvoDir, $batch); - - # and publish - if ($tarball && defined $datastore ) { - - $published = $datastore->register($batch, $dvoDir, $tarball, "IPP_PSPS", "tgz"); - } - } - } - - $ippToPspsDb->updateBatch($batchId, $expId, 1, $published, $totalDetections, $minObjId, $maxObjId); - } - - if ($initBatch) {print "* Wrote initialisation batch\n";} - - print "*\n*******************************************************************************\n\n"; - } - - close $resultsFile; - - return 1; -} - -####################################################################################### -# -# run IppToPsps -# -####################################################################################### -sub runIppToPsps { - my ($expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath) = @_; - - my $success = 0; - - if ($batchType =~ /init/) { - $success = produceInit($batchDir, $resultsFilePath); - } - elsif ($batchType =~ /det/) { - $success = produceDetections($expId, $expName, $surveyType, $batchDir, $resultsFilePath); - } - elsif ($batchType =~ /diff/) { - $success = produceDiffs($expId, $expName, $surveyType, $batchDir, $resultsFilePath); - } - elsif ($batchType =~ /test/) { - $success = produceDetections($expId, $expName, $surveyType, $batchDir, $resultsFilePath); - } - - return $success; -} - -####################################################################################### -# -# tar 'n zip a batch TODO handle errors -# -####################################################################################### -sub tarAndZipBatch { - my ($path,$batch) = @_; - - my $batchDir = "$path/$batch"; - my $tar = "$batch.tar"; - my $tarball = "$tar.gz"; - my $batchTar = "$path/$tar"; - my $batchZip = "$path/$tarball"; - - # tar - my $command = "tar -cvf $batchTar -C $path $batch"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to tar up contents of dir: $batchDir\n" and return undef; } - - # zip - $command = "gzip -c $batchTar > $batchZip"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to gzip: $batchTar\n" and return undef; } - - # remove tar file - $command = "rm $batchTar"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to remove: $batchTar\n" }; - - # remove batch dir - $command = "rm -r $batchDir"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - if (!$success) { print "* Unable to remove: $batch\n"}; - - return $tarball; -} - -####################################################################################### -# -# zip a file then delete it -# -####################################################################################### -sub zipFile { - my ($path,$origFile) = @_; - - my $file; - - # zip - my $zippedName = "$origFile.gz"; - my $command = "gzip -c $path/$origFile > $path/$zippedName"; - my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) = - run(command => $command, verbose => $verbose); - - if (!$success) { - - $file = $origFile; - print "* Unable to gzip: $path/$origFile\n"; - } - else { - - $file = $zippedName; - - # now delete original - $command = "rm -r $path/$origFile"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - - if (!$success) { print "* Unable to remove: $path/$origFile\n"}; - } - - return $file; -} - - -####################################################################################### -# -# parse results XML -# -####################################################################################### -sub parseResults { - my ($resultsFilePath, $filename, $minObjId, $maxObjId, $totalDetections) = @_; - - my $parser = XML::LibXML->new; - my $doc = $parser->parse_file($resultsFilePath); - ${$filename} = $doc->findvalue('//filename'); - ${$minObjId} = $doc->findvalue('//minObjID'); - ${$maxObjId} = $doc->findvalue('//maxObjID'); - ${$totalDetections} = $doc->findvalue('//totalDetections'); -} - - -####################################################################################### -# -# writes a batch manifest file for the one FITS file that should be in the path passed in -# -####################################################################################### -sub writeBatchManifest { - my ($path,$batch,$batchType,$surveyType,$filename,$minObjId,$maxObjId) = @_; - - use XML::Writer; - use Digest::MD5::File qw( file_md5_hex ); - - # sort out paths - my $outputPath = "$path/BatchManifest.xml"; - my $output = new IO::File(">$outputPath"); - - # create a timestamp - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time); - my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec; - - # create XML file - my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2); - $writer->xmlDecl('UTF-8'); - $writer->doctype('manifest', "", "psps-manifest.dtd"); - - # open directory to hunt for FITS files - opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0; - my @thefiles= readdir(DIR); - closedir(DIR); - - my $foundFits = 0; - foreach my $f (@thefiles) { - - if ($f =~ /\.FITS/) { - - if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;} - - # my $file = zipFile($path, $f); - - # get md5sum of file - my $md5sum = file_md5_hex("$path/$f"); - - # get size of file in bytes - my $filesize = -s "$path/$f"; - - # write manifest element TODO untidy - if($batchType eq 'det' && $filename eq $f) { - - my $pspsSurvey = undef; - if ($surveyType =~ m/^MD[0-1][0-9]$/i) {$pspsSurvey = "MDF";} - else {$pspsSurvey = $surveyType;} - - $writer->startTag('manifest', - "name" => "$batch", - "type" => $pspsBatchType, - "survey" => $pspsSurvey, - "timestamp" => "$timeStamp", - "minObjId" => $minObjId, - "maxObjId" => $maxObjId); - } - else { - - $writer->startTag('manifest', - "name" => "$batch", - "type" => $pspsBatchType, - "timestamp" => "$timeStamp"); - } - # write the tag for this batch - $writer->startTag("file", - "name" => $f, - "bytes" => $filesize, - "md5" => $md5sum); - - $writer->endTag(); - $writer->endTag(); - $writer->end(); - $foundFits = 1; - } - - } - - if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;} - return 1; -} - -####################################################################################### -# -# runs ippToPsps to produce init batch -# -####################################################################################### -sub produceInit { - my ($outputPath, $resultsFilePath) = @_; - - my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType, $resultsFilePath); - return $success; -} - -####################################################################################### -# -# runs ippToPsps for detections for this exposure -# -####################################################################################### -sub produceDetections { - my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; - - my $nebPath = $gpc1Db->getCameraStageSmfForThisDvoDb($dvoDb, $expId); - if (!$nebPath) { return 0; } - - my $success = 0; - - # get real filename from neb 'key' - my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0; - my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; - - # now write the path to this file out to temp file - my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$save_temps); - print $tempFile $realFile . "\n"; - close $tempFile; - - $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); - return $success; -} - -####################################################################################### -# -# runs ippToPsps for diffs for this exposure -# -####################################################################################### -sub produceDiffs { - my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_; - - my $rows = $gpc1Db->getDiffStageCmfs($expId); - - my $size = @{$rows}; - if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;} - - my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps); - - # loop round db results should be one file per skycell - my $success = 0; - - my $row; - foreach $row ( @{$rows} ) { - my ($nebPath) = @{$row}; - - print "NEB PATH $nebPath\n"; - - # get real filename from neb 'key' - my $fpaObjects = $ipprc->filename("PPSUB.OUTPUT.SOURCES",$nebPath) or return 0; - my $realFile = $ipprc->file_resolve($fpaObjects) or return 0; - #print "REALFILE $realFile\n"; - - # now write the path to this file out to temp file - print $tempFile $realFile . "\n"; - } - - my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath); - - close $tempFile; - - return $ret; -} - -####################################################################################### -# -# runs the ippToPsps program -# -####################################################################################### -sub runProgram { - my ($input, $output, $expid, $expName, $surveyType, $batchType, $resultsFilePath) = @_; - - # build command - my $command = "$ippToPsps"; - $command .= " -input $input"; - $command .= " -output $output"; - $command .= " -D CATDIR $fullDvoPath"; - $command .= " -config ../config"; # TODO - $command .= " -expid $expid"; - $command .= " -expname $expName"; - $command .= " -survey $surveyType"; - $command .= " -batch $batchType"; - $command .= " -results $resultsFilePath"; - - # run command - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) - = run(command => $command, verbose => $verbose); - - return $success; -} Index: branches/meh_branches/ppstack_test/ippToPsps/perl/removeFromDatastore.pl =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/perl/removeFromDatastore.pl (revision 32161) +++ (revision ) @@ -1,56 +1,0 @@ -#!/usr/bin/env perl - -use warnings; -use strict; -use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); -use Pod::Usage qw( pod2usage ); - -use ippToPsps::Datastore; - -my $start = undef; -my $end = undef; -my $product = undef; - -GetOptions( - 'start|s=s' => \$start, - 'end|e=s' => \$end, - 'product|p=s' => \$product, - ); - -my $quit = 0; -print "\n*******************************************************************************\n"; -print "* \n"; -if (@ARGV) { - $quit=1; - print "* UNKNKOWN: option @ARGV\n"; -} -if (!defined $product) { - $quit=1; - print "* REQUIRED: a datastore product name -p \n"; -} -if (!defined $start) { - $quit=1; - print "* REQUIRED: starting batch number -s \n"; -} -if (!defined $end) { - $end=99999999999; - print "* OPTIONAL: ending batch number -e (default=$end)\n"; -} -print "*\n*******************************************************************************\n"; - -if ($quit) { exit; } - -my $datastore = new ippToPsps::Datastore($product, 0, 0); - -my $i; - -for ($i=$start; $i<$end+1; $i++) { - - my $name = sprintf("B%08d", $i); - if ($datastore->remove($name)) { - print "* Successfully removed $name from datastore\n"; - } - else { - print "* Unable to remove $name from datastore\n"; - } -} Index: branches/meh_branches/ppstack_test/ippToPsps/src/Dvo.c =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/src/Dvo.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/src/Dvo.c (revision 33415) @@ -35,5 +35,5 @@ mysql_query(this->mysql, "DELETE FROM dvoDetection"); - mysql_query(this->mysql, "SELECT sourceID, externID FROM dvoMeta"); + mysql_query(this->mysql, "SELECT SOURCE_ID, EXTERN_ID FROM dvoImages"); MYSQL_RES* result = mysql_store_result(this->mysql); @@ -61,5 +61,5 @@ sprintf(sql, - "UPDATE dvoMeta SET imageID = %d,flags = %d, photcode = %d WHERE sourceID = %d AND externID = %d", + "UPDATE dvoImages SET IMAGE_ID = %d, FLAGS = %d, PHOTCODE = %d WHERE SOURCE_ID = %d AND EXTERN_ID = %d", image->imageID, image->flags, @@ -91,6 +91,5 @@ else { sprintf(sql, - "INSERT INTO dvoDetection (sourceID, imageID, ippDetectID, detectID, ippObjID, objID, flags) VALUES (%d, %d, %u, %lu, %lu, %lu, %u)", - sourceID, // sourceID + "INSERT INTO dvoDetection (imageID, ippDetectID, detectID, ippObjID, objID, flags) VALUES (%d, %u, %lu, %lu, %lu, %u)", image->imageID, // imageID dvoDetections[i].meas.detID, // ippDetectID @@ -117,5 +116,5 @@ Constructor. */ -Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dvoDatabase) { +Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dbname, const char* dvoDatabase) { Dvo* this = (Dvo*)calloc(1, sizeof(Dvo)); @@ -130,5 +129,4 @@ char user[20]; char passwd[20]; - char dbname[20]; @@ -157,6 +155,5 @@ sprintf(name, "%s", dbParams->name); value = xmlNodeListGetString(doc, dbParams->xmlChildrenNode, 1); - if (strcmp(name, "name") == 0) sprintf(dbname, "%s", value); - else if (strcmp(name, "user") == 0) sprintf(user, "%s", value); + if (strcmp(name, "user") == 0) sprintf(user, "%s", value); else if (strcmp(name, "host") == 0) sprintf(host, "%s", value); else if (strcmp(name, "password") == 0) sprintf(passwd, "%s", value); @@ -225,13 +222,12 @@ Logger* logger = new_Logger(NULL, true); - if (argc < 3) { + if (argc < 4) { logger->print(logger, MSG_ERROR, "Dvo", "Incorrect arguments\n"); - logger->print(logger, MSG_INFO, "Dvo", "Usage: ./dvo \n"); + logger->print(logger, MSG_INFO, "Dvo", "Usage: ./dvo \n"); } else { - //Dvo * dvo = new_Dvo(logger, "/data/ipp004.0/gpc1/catdirs/ThreePi.V1"); - Dvo * dvo = new_Dvo(logger, argv[1], argv[2]); - //dvo->getDetections(dvo, 33, 8345290); + + Dvo * dvo = new_Dvo(logger, argv[1], argv[2], argv[3]); dvo->getDetections(dvo); dvo->destroy(dvo); Index: branches/meh_branches/ppstack_test/ippToPsps/src/Dvo.h =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/src/Dvo.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/src/Dvo.h (revision 33415) @@ -42,5 +42,5 @@ // constructor -Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dvoDatabase); +Dvo* new_Dvo(Logger* logger, const char* configFile, const char* dbname, const char* dvoDatabase); #endif // IPPTOPSPS_DVO_H Index: branches/meh_branches/ppstack_test/ippToPsps/xmlschema/BatchManifest.xsd =================================================================== --- branches/meh_branches/ppstack_test/ippToPsps/xmlschema/BatchManifest.xsd (revision 32161) +++ branches/meh_branches/ppstack_test/ippToPsps/xmlschema/BatchManifest.xsd (revision 33415) @@ -51,5 +51,10 @@ - + + + + + + Index: branches/meh_branches/ppstack_test/ippTools/configure.ac =================================================================== --- branches/meh_branches/ppstack_test/ippTools/configure.ac (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/configure.ac (revision 33415) @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([ipptools], [1.1.66], [ipp-support@ifa.hawaii.edu]) +AC_INIT([ipptools], [1.1.71], [ipp-support@ifa.hawaii.edu]) AC_CONFIG_SRCDIR([autogen.sh]) @@ -18,5 +18,5 @@ PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0]) PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0]) -PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.66]) +PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.71]) PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) Index: branches/meh_branches/ppstack_test/ippTools/share/Makefile.am =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/Makefile.am (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/Makefile.am (revision 33415) @@ -10,4 +10,6 @@ addtool_find_sky_id_dvo.sql \ addtool_find_sky_id.sql \ + addtool_find_sky_id_multi_dvo.sql \ + addtool_find_sky_id_multi.sql \ addtool_find_stack_id_dvo.sql \ addtool_find_stack_id.sql \ @@ -16,8 +18,10 @@ addtool_find_pendingexp_stack.sql \ addtool_find_pendingexp_staticsky.sql \ + addtool_find_pendingexp_staticsky_multi.sql \ addtool_find_pendingmergeprocess.sql \ addtool_find_processedexp_cam.sql \ addtool_find_processedexp_stack.sql \ addtool_find_processedexp_staticsky.sql \ + addtool_find_processedexp_staticsky_multi.sql \ addtool_find_minidvodbprocessed.sql \ addtool_find_minidvodbrun.sql \ @@ -27,4 +31,5 @@ addtool_queue_stack_id.sql \ addtool_queue_sky_id.sql \ + addtool_queue_sky_id_multi.sql \ addtool_queue_minidvodbrun.sql \ addtool_revertminidvodbprocessed.sql \ @@ -32,4 +37,5 @@ addtool_revertprocessedexp_stack.sql \ addtool_revertprocessedexp_staticsky.sql \ + addtool_revertprocessedexp_staticsky_multi.sql \ bgtool_advancechip.sql \ bgtool_advancewarp.sql \ @@ -76,4 +82,6 @@ chiptool_definecopy.sql \ chiptool_donecleanup.sql \ + chiptool_dropprocessedimfile_update.sql \ + chiptool_dropprocessedimfile_delete.sql \ chiptool_find_rawexp.sql \ chiptool_listrun.sql \ @@ -179,4 +187,5 @@ disttool_definebyquery_raw_no_magic.sql \ disttool_definebyquery_sky.sql \ + disttool_definebyquery_sky_singlefilter.sql \ disttool_definebyquery_stack.sql \ disttool_definebyquery_warp.sql \ @@ -244,4 +253,5 @@ magictool_definebyquery_insert.sql \ magictool_definebyquery_select.sql \ + magictool_definebyquery_select_multidiff.sql \ magictool_inputs.sql \ magictool_inputskyfile.sql \ @@ -274,4 +284,5 @@ magicdstool_definebyquery_warp_bg.sql \ magicdstool_definebyquery_diff.sql \ + magicdstool_definecopy_camera.sql \ magicdstool_definecopy_chip.sql \ magicdstool_definecopy_warp.sql \ @@ -299,8 +310,19 @@ minidvodbtool_queue_minidvodb_id.sql \ minidvodbtool_revertminidvodbcopy.sql \ + mergetool_find_minidvodb_id.sql \ + mergetool_find_mergedvodbcopy.sql \ + mergetool_find_merge_id_dvo.sql \ + mergetool_find_pendingmergeprocess.sql \ + mergetool_find_pending_merge.sql \ + mergetool_find_processed.sql \ + mergetool_queue_mergecopy_id.sql \ + mergetool_queue_merge_id.sql \ + mergetool_revertmergedvodbcopy.sql \ + mergetool_revertmergedvodbprocessed.sql \ pstamptool_completedreq.sql \ pstamptool_datastore.sql \ pstamptool_getdependent.sql \ pstamptool_listjob.sql \ + pstamptool_listfile.sql \ pstamptool_pendingcleanup.sql \ pstamptool_pendingdependent.sql \ @@ -373,4 +395,8 @@ staticskytool_definebyquery_select_by_dg.sql \ staticskytool_definebyquery_inputs.sql \ + staticskytool_defineskycalrun.sql \ + staticskytool_pendingskycalrun.sql \ + staticskytool_revertskycal.sql \ + staticskytool_skycalresult.sql \ staticskytool_export_run.sql \ staticskytool_export_input.sql \ @@ -418,9 +444,15 @@ laptool_definerun.sql \ laptool_exposures.sql \ - laptool_inactiveexp.sql \ + laptool_inactiveexp.sql \ laptool_listsequence.sql \ laptool_pendingchipexp.sql \ laptool_pendingexp.sql \ laptool_pendingrun.sql \ - laptool_stacks.sql + laptool_WSdiff_check.sql \ + laptool_stacks.sql \ + vptool_find_rawexp.sql \ + vptool_pendingimfile.sql \ + vptool_pendingrun.sql \ + vptool_processedcell.sql \ + vptool_revertrun.sql Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_checkminidvodbaddrun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_checkminidvodbaddrun.sql (revision 33415) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_id, minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_id, minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group, (cnt - cnt2) as not_full, cnt as addRun_count FROM minidvodbRun Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_find_minidvodbprocessed.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_find_minidvodbprocessed.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_find_minidvodbprocessed.sql (revision 33415) @@ -1,5 +1,5 @@ SELECT minidvodbProcessed.*, - minidvodbRun.minidvodb_name + minidvodbRun.minidvodb_name, minidvodbRun.minidvodb_group FROM minidvodbProcessed Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_find_minidvodbrun2.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_find_minidvodbrun2.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_find_minidvodbrun2.sql (revision 33415) @@ -1,4 +1,4 @@ -- this selects the minidvodbs that have complete addRuns -SELECT minidvodb_name, minidvodb_path, mergedvodb_path, state, minidbg +SELECT minidvodb_name, minidvodb_path, state, minidbg AS minidvodb_group FROM minidvodbRun Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_find_pendingexp_staticsky_multi.sql (revision 33415) @@ -0,0 +1,32 @@ +SELECT + addRun.*, + staticskyResult.path_base as stageroot, + rawExp.camera, + rawExp.telescope +FROM addRun +JOIN staticskyRun + ON sky_id = stage_id +JOIN staticskyResult + USING (sky_id) +JOIN staticskyInput + USING (sky_id) +JOIN stackRun + USING (stack_id) +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpRun using(warp_id) +JOIN fakeRun using(fake_id) +JOIN camRun using(cam_id) +JOIN chipRun using(chip_id) +JOIN rawExp using (exp_id) +LEFT JOIN addProcessedExp using (add_id) +LEFT JOIN addMask + ON addRun.label = addMask.label +WHERE + staticskyRun.state = 'full' + AND stage = 'staticsky_multi' + AND ((addRun.state = 'new' AND addProcessedExp.add_id IS NULL) OR addRun.state = 'update') + AND addRun.dvodb IS NOT NULL + AND addRun.workdir IS NOT NULL + AND addMask.label IS NULL + Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_find_processedexp_staticsky_multi.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_find_processedexp_staticsky_multi.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_find_processedexp_staticsky_multi.sql (revision 33415) @@ -0,0 +1,6 @@ +SELECT + addProcessedExp.*, + addRun.workdir +FROM addProcessedExp +JOIN addRun + USING(add_id) Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_find_sky_id_multi.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_find_sky_id_multi.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_find_sky_id_multi.sql (revision 33415) @@ -0,0 +1,24 @@ +SELECT + distinct(sky_id) as stage_id, + staticskyRun.*, + staticskyResult.num_inputs +FROM + staticskyResult +join staticskyRun using (sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +LEFT JOIN (SELECT sky_id AS added_sky_id, + addRun.dvodb AS previous_dvodb + FROM addRun +JOIN staticskyRun on sky_id = stage_id + ) as foo + ON sky_id = added_sky_id + AND stage = 'staticsky_multi' + -- hook for qualifying the join on the previous_dvodb + AND %s +WHERE + staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + AND added_exp_id IS NULL + -- addtool adds checks on exposure being added to the dvodb previously Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_find_sky_id_multi_dvo.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_find_sky_id_multi_dvo.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_find_sky_id_multi_dvo.sql (revision 33415) @@ -0,0 +1,12 @@ +SELECT distinct(sky_id) as stage_id, staticskyRun.*, staticskyResult.num_inputs FROM staticskyResult +JOIN staticskyRun USING(sky_id) +join staticskyInput using(sky_id) +JOIN stackRun USING(stack_id) + +WHERE staticskyRun.state = 'full' and staticskyResult.quality = 0 + AND sky_id NOT IN (SELECT sky_id + FROM addRun + JOIN staticskyResult on staticskyResult.sky_id = addRun.stage_id + JOIN staticskyRun USING(sky_id) + WHERE addRun.stage = 'staticsky_multi' AND %s + ) Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_cam_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_cam_id.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_cam_id.sql (revision 33415) @@ -4,4 +4,5 @@ 'cam', -- stage cam_id, -- stage_id + %d, -- stage_extra1 '%s', -- state '%s', -- workdir Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_minidvodbrun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_minidvodbrun.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_minidvodbrun.sql (revision 33415) @@ -4,5 +4,4 @@ minidvodb_name, minidvodb_path, - mergedvodb_path, 'new' Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_sky_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_sky_id.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_sky_id.sql (revision 33415) @@ -4,4 +4,5 @@ 'staticsky', -- stage sky_id, -- stage_id + %d, -- stage_extra1 '%s', -- state '%s', -- workdir Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_sky_id_multi.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_sky_id_multi.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_sky_id_multi.sql (revision 33415) @@ -0,0 +1,22 @@ +INSERT INTO addRun + SELECT + 0, -- add_id + 'staticsky_multi', -- stage + sky_id, -- stage_id + %d, -- stage_extra1 + '%s', -- state + '%s', -- workdir + '%s', -- workdir_state + '%s', -- reduction + '%s', -- label + '%s', -- data_group + '%s', -- dvodb + '%s', -- note + %d, -- image_only + %d, -- minidvodb + '%s', -- minidvodb_group + '%s' -- minidvodb_name + FROM staticskyRun + WHERE + staticskyRun.state = 'full' + AND staticskyRun.sky_id = %lld Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_stack_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_stack_id.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_queue_stack_id.sql (revision 33415) @@ -4,4 +4,5 @@ 'stack', -- stage stack_id, -- stage_id + %d, --stage_extra1 '%s', -- state '%s', -- workdir Index: branches/meh_branches/ppstack_test/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/addtool_revertprocessedexp_staticsky_multi.sql (revision 33415) @@ -0,0 +1,7 @@ +DELETE FROM addProcessedExp +USING addProcessedExp, addRun +WHERE + addRun.add_id = addProcessedExp.add_id + AND addProcessedExp.fault != 0 + AND addRun.state = 'new' + AND addRun.stage = 'staticsky_multi' Index: branches/meh_branches/ppstack_test/ippTools/share/bgtool_definechip.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/bgtool_definechip.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/bgtool_definechip.sql (revision 33415) @@ -5,3 +5,2 @@ LEFT JOIN chipBackgroundRun USING(chip_id) WHERE chipRun.state = 'full' - AND chipRun.magicked > 0 -- require destreaked to stop race condition in destreaking Index: branches/meh_branches/ppstack_test/ippTools/share/camtool_find_pendingimfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/camtool_find_pendingimfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/camtool_find_pendingimfile.sql (revision 33415) @@ -2,6 +2,7 @@ camRun.cam_id, chipProcessedImfile.*, - (IF(exp_time IS NOT NULL AND deteff_magref IS NOT NULL, - (2.5 * LOG10(exp_time) + chipProcessedImfile.deteff_magref), NULL)) + (IF(rawExp.exp_time IS NOT NULL AND deteff_magref IS NOT NULL, + (2.5 * LOG10(rawExp.exp_time) + chipProcessedImfile.deteff_magref), + NULL)) AS deteff_inst, rawExp.exp_name, @@ -13,7 +14,9 @@ USING (chip_id) JOIN chipProcessedImfile - USING (chip_id) + USING (chip_id, exp_id) JOIN rawExp - ON chipProcessedImfile.exp_id = rawExp.exp_id + USING(exp_id) +JOIN rawImfile + USING(exp_id, class_id) LEFT JOIN camProcessedExp USING(cam_id) @@ -22,2 +25,3 @@ WHERE camMask.label IS NULL + AND rawImfile.ignored = 0 Index: branches/meh_branches/ppstack_test/ippTools/share/camtool_find_processedexp.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/camtool_find_processedexp.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/camtool_find_processedexp.sql (revision 33415) @@ -3,4 +3,5 @@ camRun.workdir, camRun.label, + camRun.data_group, camRun.state, camRun.magicked, Index: branches/meh_branches/ppstack_test/ippTools/share/chiptool_change_exp_state.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/chiptool_change_exp_state.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/chiptool_change_exp_state.sql (revision 33415) @@ -14,4 +14,5 @@ WHERE chipProcessedImfile.chip_id = chipRun.chip_id + AND chipProcessedImfile.quality = 0 AND data_state != '%s' ) = 0 Index: branches/meh_branches/ppstack_test/ippTools/share/chiptool_definecopy.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/chiptool_definecopy.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/chiptool_definecopy.sql (revision 33415) @@ -1,5 +1,8 @@ SELECT rawExp.*, - newExp.label + newExp.label, + chipOld.workdir AS old_workdir, + chipOld.data_group AS old_data_group, + chipOld.state AS old_state FROM chipRun AS chipOld JOIN rawExp USING(exp_id) Index: branches/meh_branches/ppstack_test/ippTools/share/chiptool_dropprocessedimfile_delete.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/chiptool_dropprocessedimfile_delete.sql (revision 33415) @@ -0,0 +1,7 @@ +DELETE FROM magicDSFile + USING magicDSRun, magicDSFile, chipRun, chipProcessedImfile +WHERE chipRun.chip_id = chipProcessedImfile.chip_id + AND magicDSRun.stage ='chip' + AND magicDSRun.stage_id = chipRun.chip_id + AND magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + AND magicDSFile.component = chipProcessedImfile.class_id Index: branches/meh_branches/ppstack_test/ippTools/share/chiptool_dropprocessedimfile_update.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/chiptool_dropprocessedimfile_update.sql (revision 33415) @@ -0,0 +1,5 @@ +UPDATE chipProcessedImfile + SET quality = %d, + data_state = 'full', + magicked = 0, + fault = 0 Index: branches/meh_branches/ppstack_test/ippTools/share/chiptool_pendingcleanupimfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 33415) @@ -14,5 +14,5 @@ WHERE ((chipRun.state = 'goto_cleaned' AND (chipProcessedImfile.data_state = 'full' - OR chipProcessedImfile.data_state = 'update')) + OR chipProcessedImfile.data_state = 'update') AND chipProcessedImfile.quality = 0) OR (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed') Index: branches/meh_branches/ppstack_test/ippTools/share/chiptool_setimfiletoupdate.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/chiptool_setimfiletoupdate.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/chiptool_setimfiletoupdate.sql (revision 33415) @@ -1,7 +1,7 @@ UPDATE chipRun JOIN chipProcessedImfile USING(chip_id) - LEFT JOIN magicDSRun ON (stage_id = chip_id AND stage = 'chip') - LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id - AND component = class_id) + -- LEFT JOIN magicDSRun ON (stage_id = chip_id AND stage = 'chip') + -- LEFT JOIN magicDSFile ON (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id + -- AND component = class_id) SET chipRun.state = 'update', chipProcessedImfile.data_state = 'update', @@ -11,7 +11,8 @@ AND (chipRun.state = 'cleaned' OR chipRun.state = 'update') AND (chipProcessedImfile.data_state = 'cleaned') + AND (chipProcessedImfile.quality = 0) -- don't queue update if the associated magicDSFile exists and isn't cleaned - AND (chipRun.magicked = 0 - OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') - AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update')) - ) + -- AND (chipRun.magicked = 0 + -- OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update') + -- AND (magicDSFile.data_state = 'cleaned' OR magicDSFile.data_state = 'update')) + -- ) Index: branches/meh_branches/ppstack_test/ippTools/share/difftool_change_run_state.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/difftool_change_run_state.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/difftool_change_run_state.sql (revision 33415) @@ -14,3 +14,4 @@ diffSkyfile.diff_id = diffRun.diff_id AND data_state != '%s' + AND diffSkyfile.quality = 0 ) = 0 Index: branches/meh_branches/ppstack_test/ippTools/share/difftool_pendingcleanuprun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/difftool_pendingcleanuprun.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/difftool_pendingcleanuprun.sql (revision 33415) @@ -2,5 +2,5 @@ -- all of this is just to get the camera used for the diff run SELECT DISTINCT * FROM - (SELECT + (SELECT DISTINCT diffRun.diff_id, rawExp.camera, @@ -29,6 +29,7 @@ diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') + @INNERLIMITS@ UNION - SELECT + SELECT DISTINCT diffRun.diff_id, rawExp.camera, @@ -57,6 +58,7 @@ diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') + @INNERLIMITS@ UNION - SELECT + SELECT DISTINCT diffRun.diff_id, rawExp.camera, @@ -85,6 +87,7 @@ diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') + @INNERLIMITS@ UNION - SELECT + SELECT DISTINCT diffRun.diff_id, rawExp.camera, @@ -113,4 +116,5 @@ diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') + @INNERLIMITS@ ) as Foo WHERE 1 Index: branches/meh_branches/ppstack_test/ippTools/share/difftool_pendingcleanupskyfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/difftool_pendingcleanupskyfile.sql (revision 33415) @@ -9,5 +9,5 @@ USING(diff_id) WHERE - ((diffRun.state = 'goto_cleaned' AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update')) + ((diffRun.state = 'goto_cleaned' AND (diffSkyfile.data_state = 'full' OR diffSkyfile.data_state = 'update') AND diffSkyfile.quality = 0) OR (diffRun.state = 'goto_scrubbed' AND diffSkyfile.data_state != 'scrubbed') Index: branches/meh_branches/ppstack_test/ippTools/share/difftool_setskyfiletoupdate.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/difftool_setskyfiletoupdate.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/difftool_setskyfiletoupdate.sql (revision 33415) @@ -8,2 +8,3 @@ AND (diffRun.state = 'cleaned' OR diffRun.state = 'update') AND (diffSkyfile.data_state = 'cleaned' OR diffSkyfile.data_state = 'update') + AND (diffSkyfile.quality = 0) Index: branches/meh_branches/ppstack_test/ippTools/share/difftool_todiffskyfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/difftool_todiffskyfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/difftool_todiffskyfile.sql (revision 33415) @@ -77,5 +77,5 @@ OR (warpSkyfile.data_state = 'full' AND warpSkyfile.fault = 0 - AND warpSkyfile.magicked >= 0 +-- AND warpSkyfile.magicked >= 0 AND warpSkyfile.quality = 0)) -- Ensure input stacks are available @@ -88,5 +88,5 @@ OR (warpTemplateSkyfile.data_state = 'full' AND warpTemplateSkyfile.fault = 0 - AND warpSkyfile.magicked >= 0 +-- AND warpSkyfile.magicked >= 0 AND warpTemplateSkyfile.quality = 0)) -- Ensure template stacks are available Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_raw_no_magic.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_raw_no_magic.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_raw_no_magic.sql (revision 33415) @@ -14,5 +14,4 @@ JOIN rcInterest USING(target_id) LEFT JOIN distRun ON distRun.stage_id = exp_id AND distRun.target_id = distTarget.target_id -WHERE distRun.dist_id IS NULL -- no existing distRun for this exposure - AND distTarget.state = 'enabled' -- target and intrest are enabled +WHERE distTarget.state = 'enabled' -- target and intrest are enabled AND rcInterest.state = 'enabled' Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_sky.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_sky.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_sky.sql (revision 33415) @@ -22,4 +22,5 @@ WHERE distTarget.state = 'enabled' AND rcInterest.state = 'enabled' + AND distTarget.filter = 'multi' AND distRun.dist_id IS NULL AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_sky_singlefilter.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_sky_singlefilter.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_definebyquery_sky_singlefilter.sql (revision 33415) @@ -0,0 +1,28 @@ +SELECT + 'sky' as stage, + staticskyRun.sky_id AS stage_id, + CAST(0 AS SIGNED) AS magicked, + -- run tag in the form 'sky.$skycell_id.$stack_id' + CONCAT_WS('.', 'sky', stackRun.skycell_id, convert(staticskyRun.sky_id, CHAR)) as run_tag, + staticskyRun.label, + staticskyRun.data_group, + distTarget.dist_group, + distTarget.target_id, + distTarget.clean +FROM staticskyRun +JOIN staticskyResult USING(sky_id) +JOIN staticskyInput USING(sky_id) +JOIN stackRun using(stack_id) +JOIN distTarget ON distTarget.stage = 'sky' + AND staticskyRun.dist_group = distTarget.dist_group +JOIN rcInterest USING(target_id) +LEFT JOIN distRun ON (distRun.stage_id = sky_id) + AND distRun.target_id = distTarget.target_id + -- JOIN hook %s +WHERE distTarget.state = 'enabled' + AND rcInterest.state = 'enabled' + AND distRun.dist_id IS NULL + AND ((staticskyRun.state = 'full') OR (distTarget.clean AND staticskyRun.state = 'cleaned')) + AND stackRun.filter = distTarget.filter + -- we shouldn't need to check fault. If faulted it shouldn't be full + AND (staticskyResult.fault = 0 AND staticskyResult.quality = 0) Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_listinterests.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_listinterests.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_listinterests.sql (revision 33415) @@ -9,4 +9,5 @@ stage, clean, + target_id, distTarget.state as target_state FROM rcInterest Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_pending_chip_bg.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_pending_chip_bg.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_pending_chip_bg.sql (revision 33415) @@ -14,5 +14,5 @@ chipBackgroundRun.state, chipBackgroundRun.state AS data_state, - 0 AS quality, + chipBackgroundImfile.quality AS quality, distRun.no_magic, chipBackgroundImfile.magicked, Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_pending_raw.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_pending_raw.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_pending_raw.sql (revision 33415) @@ -43,4 +43,5 @@ -- AND (rawExp.magicked OR distRun.no_magic) AND rawExp.magicked + AND rawImfile.ignored = 0 -- need to have magicked the chip image which makes the camera mask AND chipProcessedImfile.magicked != 0 @@ -86,4 +87,5 @@ AND distComponent.dist_id IS NULL AND (distRun.no_magic) + AND rawImfile.ignored = 0 AND (Label.active OR Label.active IS NULL) UNION @@ -131,4 +133,5 @@ AND chipProcessedImfile.magicked != 0 AND camRun.magicked > 0 + AND rawImfile.ignored = 0 AND (Label.active OR Label.active IS NULL) UNION Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_processedcomponent.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_processedcomponent.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_processedcomponent.sql (revision 33415) @@ -10,4 +10,5 @@ name, distComponent.state, + no_magic, distComponent.fault FROM distRun Index: branches/meh_branches/ppstack_test/ippTools/share/disttool_toadvance.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/disttool_toadvance.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/disttool_toadvance.sql (revision 33415) @@ -26,4 +26,5 @@ AND distRun.fault = 0 AND distRun.stage = 'raw' + AND rawImfile.ignored = 0 GROUP BY distRun.dist_id, Index: branches/meh_branches/ppstack_test/ippTools/share/laptool_WSdiff_check.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/laptool_WSdiff_check.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/laptool_WSdiff_check.sql (revision 33415) @@ -0,0 +1,24 @@ +SELECT warp_id,warp_state,W.skycell_id,stack_id,stack_state,S.seq_id + FROM +( SELECT warp_id,skycell_id,seq_id,warpRun.state AS warp_state + FROM warpSkyfile + JOIN warpRun USING(warp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) + JOIN lapExp USING(exp_id,chip_id) + JOIN lapRun USING(lap_id) + WHERE 1 + @WHERE@ +) AS W +LEFT JOIN +( SELECT stack_id,skycell_id,seq_id,stackRun.state AS stack_state + FROM stackRun + JOIN stackAssociationMap USING(stack_id) + JOIN lapRun ON (stackAssociationMap.sass_id = lapRun.final_sass_id) + ) AS S +ON ( (W.seq_id = S.seq_id) + AND (W.skycell_id = S.skycell_id) ) + + Index: branches/meh_branches/ppstack_test/ippTools/share/laptool_definerun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/laptool_definerun.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/laptool_definerun.sql (revision 33415) @@ -1,3 +1,8 @@ -SELECT DISTINCT want.exp_id, have.chip_id, false as private, true as active, false as pairwise +SELECT DISTINCT + want.exp_id, + have.chip_id, + COALESCE(have.private,false) as private, + true as active, + false as pairwise FROM (SELECT exp_id FROM rawExp @@ -8,15 +13,21 @@ ) AS want LEFT JOIN - (SELECT exp_id,MAX(chip_id) AS chip_id, chipRun.state + (SELECT + exp_id, + MAX(chip_id) AS chip_id, + chipRun.state, + lapExp.private FROM lapExp JOIN chipRun USING(exp_id,chip_id) - where private IS FALSE - AND chip_id IS NOT NULL + JOIN lapRun USING(lap_id) + where chip_id IS NOT NULL AND - ( +-- This is the restriction to only draw from the current lapSequence + @CHIPWHERE@ +-- ( -- (active = TRUE) OR - (chipRun.state = 'full') OR - (chipRun.state = 'new') + -- (chipRun.state = 'full') OR + -- (chipRun.state = 'new') -- when we can do updates, put that here. - ) +-- ) GROUP BY exp_id ) AS have USING(exp_id) Index: branches/meh_branches/ppstack_test/ippTools/share/laptool_pendingexp.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/laptool_pendingexp.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/laptool_pendingexp.sql (revision 33415) @@ -3,7 +3,9 @@ lapRun.dist_group, lapRun.registered, lapRun.fault, lapRun.quick_sass_id, lapRun.final_sass_id, exp_id, chip_id, pair_id, private, pairwise, active, lapExp.data_state, - dateobs, object, comment + dateobs, object, comment, + chipRun.state AS chip_state FROM lapRun JOIN lapExp USING(lap_id) JOIN rawExp USING(exp_id,filter) + LEFT JOIN chipRun USING(exp_id,chip_id) WHERE active IS TRUE AND lapRun.fault = 0 -- lap_id restriction here. Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_definecopy_camera.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_definecopy_camera.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_definecopy_camera.sql (revision 33415) @@ -0,0 +1,31 @@ +SELECT DISTINCT -- DISTINCT because we are going though diffInputSkyfile + magicRun.magic_id, + chipRun.exp_id, + 'camera' AS stage, + camRun.cam_id AS stage_id, + camRun.data_group, + camRun.cam_id, + camRun.label, + CAST(NULL AS SIGNED) AS inv_magic_id +FROM magicRun + JOIN magicMask USING(magic_id) + JOIN diffRun USING(diff_id) + JOIN diffInputSkyfile USING(diff_id) + JOIN warpRun AS oldWarpRun + ON (!magicRun.inverse AND diffInputSkyfile.warp1 = oldWarpRun.warp_id) + OR ( magicRun.inverse AND diffInputSkyfile.warp2 = oldWarpRun.warp_id) + -- This ends the usual JOIN down to the raw level + JOIN rawExp + ON rawExp.exp_id = magicRun.exp_id + JOIN chipRun + ON chipRun.exp_id = rawExp.exp_id + JOIN camRun + ON camRun.chip_id = chipRun.chip_id + LEFT JOIN magicDSRun + ON magicRun.magic_id = magicDSRun.magic_id + AND magicDSRun.stage = 'camera' + AND magicDSRun.label = '%s' +WHERE magicRun.state = 'full' + -- HOOK: NEWLINE HERE TO ACTIVATE %s AND magicDSRun.magic_ds_id IS NULL + AND camRun.magicked = 0 + AND camRun.state = 'full' Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_getskycells.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_getskycells.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_getskycells.sql (revision 33415) @@ -5,5 +5,6 @@ diffSkyfile.skycell_id, diffSkyfile.path_base, - diffSkyfile.data_state + diffSkyfile.data_state, + diffSkyfile.quality FROM magicDSRun JOIN magicRun USING(magic_id) @@ -37,5 +38,6 @@ diffSkyfile.skycell_id, diffSkyfile.path_base, - diffSkyfile.data_state + diffSkyfile.data_state, + diffSkyfile.quality FROM magicDSRun JOIN magicRun USING(magic_id) Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_camera.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_camera.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_camera.sql (revision 33415) @@ -14,4 +14,5 @@ stage_id, 'exposure' AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, camProcessedExp.path_base, Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_chip.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_chip.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_chip.sql (revision 33415) @@ -14,4 +14,5 @@ stage_id, class_id AS component, + 0 AS mismatched_tess, chipProcessedImfile.uri, chipProcessedImfile.path_base, Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_diff.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_diff.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_diff.sql (revision 33415) @@ -15,4 +15,5 @@ diffRun.magicked, diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, @@ -63,4 +64,5 @@ diffRun.magicked, diffSkyfile.skycell_id AS component, + 0 AS mismatched_tess, CAST(NULL AS CHAR(255)) AS uri, diffSkyfile.path_base, Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_raw.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_raw.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_raw.sql (revision 33415) @@ -14,4 +14,5 @@ rawExp.magicked, class_id as component, + 0 AS mismatched_tess, rawImfile.uri AS uri, -- XXX: replace this with rawImfile.path_base once it exists Index: branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_warp.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_warp.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/magicdstool_todestreak_warp.sql (revision 33415) @@ -14,4 +14,6 @@ warpRun.magicked, warpSkyfile.skycell_id as component, + diffRun.tess_id != warpRun.tess_id AS mismatched_tess, + diffRun.tess_id as diff_tess_id, warpSkyfile.uri, warpSkyfile.path_base, @@ -27,4 +29,5 @@ JOIN magicMask USING (magic_id) JOIN magicRun USING (magic_id) +JOIN diffRun USING(diff_id) JOIN warpRun ON warp_id = stage_id JOIN warpSkyfile USING(warp_id) Index: branches/meh_branches/ppstack_test/ippTools/share/magictool_definebyquery_select_multidiff.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/magictool_definebyquery_select_multidiff.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/magictool_definebyquery_select_multidiff.sql (revision 33415) @@ -0,0 +1,53 @@ +-- Get a list of exposures on which magic may be performed +SELECT DISTINCT + chipRun.exp_id, + MAX(diffWarps.diff_id) AS diff_id, + -- The following trick pulls out the 'inverse' value for the maximum diff_id + CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id DESC), ',', 1), UNSIGNED) AS inverse, + diff_data_group +FROM ( + -- Forward diffs + SELECT + diffRun.diff_id, + warp1 AS warp_id, + 0 AS inverse, + diffRun.data_group AS diff_data_group + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE diffInputSkyfile.warp1 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.magicked = 0 + AND diffSkyfile.quality = 0 + -- diff WHERE hook %s + UNION + -- Backward diffs + SELECT + diffRun.diff_id, + warp2 AS warp_id, + 1 AS inverse, + diffRun.data_group AS diff_data_group + FROM diffRun + JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile USING(diff_id, skycell_id) + WHERE diffInputSkyfile.warp2 IS NOT NULL + AND diffRun.exposure = 1 + AND diffRun.bothways = 1 + AND diffRun.magicked = 0 + AND diffSkyfile.quality = 0 + -- diff WHERE hook %s + ) AS diffWarps +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING(chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN + (SELECT magic_id, exp_id, diff_id, label + FROM magicRun + -- rerun hook %s + ) AS oldMagicRun + ON oldMagicRun.exp_id = chipRun.exp_id + AND oldMagicRun.diff_id = diffWarps.diff_id +-- WHERE hook %s +GROUP BY chipRun.exp_id Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_merge_id_dvo.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_merge_id_dvo.sql (revision 33415) @@ -0,0 +1,9 @@ +SELECT mergedvodbRun.* FROM mergedvodbRun +JOIN mergedvodbProcessed USING(merge_id) +WHERE ( mergedvodbRun.state = 'merged' OR mergedvodbRun.state = 'full') + AND mergedvodbProcessed.fault = 0 + AND merge_id NOT IN (SELECT merge_id + FROM mergedvodbCopy + JOIN mergedvodbRun USING(mergedvodb_id) + WHERE %s + ) Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_mergedvodbcopy.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_mergedvodbcopy.sql (revision 33415) @@ -0,0 +1,7 @@ +SELECT + mergedvodbCopy.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbCopy + JOIN mergedvodbRun using (mergedvodb_id) + + Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_minidvodb_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_minidvodb_id.sql (revision 33415) @@ -0,0 +1,8 @@ +SELECT minidvodbRun.* FROM minidvodbRun +JOIN minidvodbProcessed USING(minidvodb_id) +WHERE minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodb_id NOT IN (SELECT minidvodb_id + FROM mergedvodbRun + JOIN minidvodbRun USING(minidvodb_id) + WHERE %s + ) Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_pending_merge.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_pending_merge.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_pending_merge.sql (revision 33415) @@ -0,0 +1,14 @@ +SELECT + mergedvodbRun.*, + minidvodbRun.minidvodb_path +FROM minidvodbRun +JOIN mergedvodbRun + USING(minidvodb_id) +LEFT JOIN mergedvodbProcessed + USING(merge_id) +WHERE + (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) + AND mergedvodbRun.mergedvodb IS NOT NULL + AND mergedvodbRun.mergedvodb_path IS NOT NULL + + Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_pendingmergeprocess.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_pendingmergeprocess.sql (revision 33415) @@ -0,0 +1,5 @@ +SELECT mergedvodbRun.* +FROM mergedvodbRun +LEFT JOIN mergedvodbProcessed +USING(merge_id) +WHERE (mergedvodbRun.state = 'new' AND mergedvodbProcessed.merge_id IS NULL) Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_processed.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_processed.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_find_processed.sql (revision 33415) @@ -0,0 +1,7 @@ +SELECT + mergedvodbProcessed.*, + mergedvodbRun.mergedvodb_path +FROM mergedvodbProcessed +JOIN mergedvodbRun + USING(merge_id) + Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_merge_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_merge_id.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_merge_id.sql (revision 33415) @@ -0,0 +1,11 @@ +INSERT INTO mergedvodbRun + SELECT + 0, -- merge_id + minidvodb_id, -- minidvodb_id + '%s', -- mergedvodb + '%s', -- mergedvodb_path + '%s', -- state + FROM minidvodbRun join minidvodbProcessed using (minidvodb_id) + WHERE + minidvodbRun.state = 'merged' and minidvodbProcessed.fault = 0 + AND minidvodbRun.minidvodb_id = %lld Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_mergecopy_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_mergecopy_id.sql (revision 33415) @@ -0,0 +1,16 @@ +INSERT INTO mergedvodbCopy + SELECT + 0, -- mergedvodbcopy_id + merge_id, -- merge_id + '%s', -- mergedvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM mergedvodbRun + WHERE + (mergedvodbRun.state = 'merged' + OR mergedvodbRun.state = 'full') + AND mergedvodbRun.merge_id = %lld Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_minidvodb_id.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_queue_minidvodb_id.sql (revision 33415) @@ -0,0 +1,15 @@ +INSERT INTO minidvodbCopy + SELECT + 0, -- minidvodbcopy_id + minidvodb_id, -- minidvodb_id + '%s', -- minidvodb_rsync_path + '%s', -- destination_host + %d, -- fault + '%s', -- state + %s, -- epoch + %f -- dtime + + FROM minidvodbRun + WHERE + minidvodbRun.state = 'merged' + AND minidvodbRun.minidvodb_id = %lld Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_revertmergedvodbcopy.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_revertmergedvodbcopy.sql (revision 33415) @@ -0,0 +1,7 @@ +UPDATE mergedvodbCopy + -- SET minidvodbCopy.state = 'new', + SET mergedvodbCopy.fault = 0 +WHERE + mergedvodbCopy.state = 'new' + AND mergedvodbCopy.fault >0 + Index: branches/meh_branches/ppstack_test/ippTools/share/mergetool_revertmergedvodbprocessed.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/mergetool_revertmergedvodbprocessed.sql (revision 33415) @@ -0,0 +1,6 @@ +DELETE FROM mergedvodbProcessed +USING mergedvodbProcessed, mergedvodbRun +WHERE + mergedvodbProcessed.merge_id = mergedvodbRun.merge_id + AND mergedvodbProcessed.fault != 0 + AND mergedvodbRun.state = 'new' Index: branches/meh_branches/ppstack_test/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/minidvodbtool_find_minidvodb_id_dvo.sql (revision 33415) @@ -1,5 +1,6 @@ SELECT minidvodbRun.* FROM minidvodbRun JOIN minidvodbProcessed USING(minidvodb_id) -WHERE minidvodbRun.state = 'merged' +WHERE minidvodbRun.state = 'merged' + AND minidvodbProcessed.fault = 0 AND minidvodb_id NOT IN (SELECT minidvodb_id FROM minidvodbCopy Index: branches/meh_branches/ppstack_test/ippTools/share/pstamptool_listfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pstamptool_listfile.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/pstamptool_listfile.sql (revision 33415) @@ -0,0 +1,5 @@ +SELECT + pstampFile.*, + pstampJob.req_id +FROM pstampFile + JOIN pstampJob using(job_id) Index: branches/meh_branches/ppstack_test/ippTools/share/pstamptool_pendingjob.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pstamptool_pendingjob.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/pstamptool_pendingjob.sql (revision 33415) @@ -1,4 +1,7 @@ -SELECT pstampJob.*, - IFNULL(Label.priority, 10000) AS priority +SELECT * FROM +(SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority, + 0 AS myParent, + 0 AS runningChildren FROM pstampJob JOIN pstampRequest USING(req_id) @@ -9,4 +12,37 @@ AND pstampJob.state = 'run' AND pstampJob.fault = 0 + AND NOT is_parent AND (dep_id = 0 OR pstampDependent.state = 'full') AND (Label.active OR Label.active IS NULL) + -- where hook 1 %s +UNION +SELECT * FROM + (SELECT pstampJob.*, + IFNULL(Label.priority, 10000) AS priority + FROM pstampJob + JOIN pstampRequest USING(req_id) + LEFT JOIN pstampDependent USING(dep_id) + LEFT JOIN Label ON pstampRequest.label = Label.label + WHERE pstampRequest.state = 'run' + AND pstampRequest.fault = 0 + AND pstampJob.state = 'run' + AND pstampJob.fault = 0 + AND is_parent + AND (dep_id = 0 OR pstampDependent.state = 'full') + AND (Label.active OR Label.active IS NULL) + -- where hook 2 %s + ) as parentJob + LEFT JOIN + (SELECT parent_id AS myParent, + count(job_id) as runningChildren + FROM pstampRequest JOIN pstampJob USING(req_id) + WHERE pstampRequest.state ='run' + AND pstampJob.state = 'run' + AND pstampJob.parent_id + GROUP BY parent_id + ) as childJobs + ON childJobs.myParent = parentJob.job_id + WHERE runningChildren IS NULL +) AS theQuery + + Index: branches/meh_branches/ppstack_test/ippTools/share/pstamptool_stopdependentjob.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pstamptool_stopdependentjob.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/pstamptool_stopdependentjob.sql (revision 33415) @@ -5,5 +5,6 @@ SET pstampJob.state = 'stop', pstampJob.fault = %d, - pstampDependent.fault = %d + pstampDependent.fault = %d, + pstampDependent.state = '%s' WHERE pstampJob.state = 'run' AND pstampDependent.state = 'new' Index: branches/meh_branches/ppstack_test/ippTools/share/pubtool_definerun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pubtool_definerun.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/pubtool_definerun.sql (revision 33415) @@ -15,4 +15,5 @@ JOIN diffRun JOIN diffInputSkyfile USING(diff_id) + JOIN diffSkyfile using(diff_id) JOIN warpRun ON warpRun.warp_id = diffInputSkyfile.warp1 -- Only JOINing input, not reference! JOIN fakeRun USING(fake_id) @@ -30,4 +31,5 @@ ) ) + AND diffSkyfile.quality = 0 -- WHERE hook %s UNION Index: branches/meh_branches/ppstack_test/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pxadmin_create_tables.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/pxadmin_create_tables.sql (revision 33415) @@ -190,7 +190,8 @@ magicked BIGINT, PRIMARY KEY(exp_id), + KEY(end_stage), + KEY(fault), + KEY(state), KEY(exp_name), - KEY(end_stage), - KEY(fault), FOREIGN KEY(exp_id) REFERENCES newExp(exp_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -554,6 +555,5 @@ KEY(workdir_state), KEY(label), - KEY(data_group), - INDEX(add_id, cam_id) + KEY(data_group) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1157,6 +1157,6 @@ tess_id VARCHAR(64), diff_skyfile_id BIGINT AUTO_INCREMENT, + PRIMARY KEY(diff_skyfile_id), KEY(diff_id, skycell_id), - KEY(diff_skyfile_id), KEY(warp1), KEY(warp2), @@ -1442,5 +1442,8 @@ fault SMALLINT, PRIMARY KEY(req_id), - KEY(req_id) + KEY(req_id), + KEY(state), + KEY(fault), + KEY(label) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1456,6 +1459,15 @@ options BIGINT, dep_id BIGINT, + fault_count INT, + parent_id BIGINT DEFAULT NULL, + is_parent TINYINT DEFAULT 0, PRIMARY KEY(job_id, req_id), KEY(job_id), + KEY(req_id), + KEY(state), + KEY(fault), + KEY(dep_id), + KEY(parent_id), + KEY(is_parent), FOREIGN KEY(req_id) REFERENCES pstampRequest(req_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1468,7 +1480,7 @@ component VARCHAR(64), imagedb VARCHAR(64), + outdir VARCHAR(255), rlabel VARCHAR(64), need_magic TINYINT, - outdir VARCHAR(255), fault SMALLINT, fault_count INT, @@ -1477,5 +1489,15 @@ KEY(stage), KEY(stage_id), + KEY(component), + KEY(imagedb), KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE pstampFile ( + file_id BIGINT AUTO_INCREMENT, + job_id BIGINT NOT NULL, + path VARCHAR(255), + PRIMARY KEY(file_id), + KEY(job_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1956,5 +1978,4 @@ minidvodb_group VARCHAR(64) NOT NULL, minidvodb_path VARCHAR(255) NOT NULL, - mergedvodb_path VARCHAR(255) NOT NULL, state VARCHAR(64) NOT NULL, creation_date TIMESTAMP, @@ -1964,14 +1985,12 @@ CREATE TABLE minidvodbProcessed ( minidvodb_id BIGINT, - merge_order BIGINT, dtime_resort FLOAT, dtime_relphot FLOAT, - dtime_verify FLOAT, - dtime_merge FLOAT, - dtime_script, - epoch TIMESTAMP, - mergedvodb_path VARCHAR(255) NOT NULL, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, fault SMALLINT NOT NULL, - KEY(minidvodb_id), CONSTRAINT UNIQUE(minidvodb_id), FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun (minidvodb_id) + UNIQUE KEY (minidvodb_id), + KEY(minidvodb_id), + FOREIGN KEY(minidvodb_id) REFERENCES minidvodbRun(minidvodb_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; @@ -1991,5 +2010,120 @@ ) ENGINE=innodb DEFAULT CHARSET=latin1; - +CREATE TABLE mergedvodbRun ( + merge_id BIGINT AUTO_INCREMENT, + minidvodb_id BIGINT, + mergedvodb VARCHAR(64) NOT NULL, + mergedvodb_path VARCHAR(255) NOT NULL, + state VARCHAR(64) NOT NULL, + creation_date TIMESTAMP, + PRIMARY KEY(merge_id), KEY(minidvodb_id), KEY(mergedvodb), INDEX(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbProcessed ( + merge_id BIGINT, + merge_order BIGINT, + dtime_verify FLOAT, + dtime_merge FLOAT, + dtime_script FLOAT, + epoch timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + fault SMALLINT NOT NULL, + UNIQUE KEY (merge_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE mergedvodbCopy ( + mergedvodbcopy_id BIGINT AUTO_INCREMENT, + merge_id BIGINT, + mergedvodb_rsync_path VARCHAR(255) NOT NULL, + destination_host VARCHAR(255) NOT NULL, + fault SMALLINT NOT NULL, + state VARCHAR(64) NOT NULL, + epoch TIMESTAMP, + dtime FLOAT, + PRIMARY KEY(mergedvodbcopy_id), + KEY(merge_id), + FOREIGN KEY(merge_id) REFERENCES mergedvodbRun(merge_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpRun ( + vp_id BIGINT AUTO_INCREMENT, + exp_id BIGINT, + state VARCHAR(64), + label VARCHAR(64), + data_group VARCHAR(64), + workdir VARCHAR(255), + note VARCHAR(255), + dest_id BIGINT, + outroot VARCHAR(255), + dtime_script FLOAT, + hostname VARCHAR(64), + fault SMALLINT NOT NULL DEFAULT 0, + PRIMARY KEY(vp_id), + KEY(exp_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(fault), + INDEX(vp_id, exp_id), + FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE vpProcessedCell ( + vp_id BIGINT, + class_id VARCHAR(64), + cell_id VARCHAR(64), + dtime_photom FLOAT, + quality SMALLINT NOT NULL DEFAULT 0, + path_base VARCHAR(255), + fault SMALLINT NOT NULL DEFAULT 0, + KEY(quality), + KEY(fault), + FOREIGN KEY(vp_id) REFERENCES vpRun(vp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE skycalRun ( + skycal_id BIGINT AUTO_INCREMENT, -- unique identifier + sky_id BIGINT, + stack_id BIGINT, + state VARCHAR(64) NOT NULL, -- state of run (new, full, etc.) + workdir VARCHAR(255) NOT NULL, -- working directory + label VARCHAR(64), -- processing label + data_group VARCHAR(64), -- group for data + dist_group VARCHAR(64), -- group for distribution + reduction VARCHAR(64), -- reduction class (for altering recipe) + registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered + note VARCHAR(255), -- note + PRIMARY KEY(skycal_id), + KEY(state), + KEY(label), + KEY(data_group), + KEY(dist_group), + KEY(sky_id), + KEY(stack_id), + FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id), + FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +-- Result of sky calibration analysis +CREATE TABLE skycalResult ( + skycal_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + dtime_astrom FLOAT, + sigma_ra FLOAT, + sigma_dec FLOAT, + n_astrom INT, + zpt_obs FLOAT, + zpt_stdev FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(64), + fault SMALLINT NOT NULL, + PRIMARY KEY(skycal_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; Index: branches/meh_branches/ppstack_test/ippTools/share/pxadmin_drop_tables.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pxadmin_drop_tables.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/pxadmin_drop_tables.sql (revision 33415) @@ -86,4 +86,5 @@ DROP TABLE IF EXISTS publishDone; DROP TABLE IF EXISTS pstampDependent; +DROP TABLE IF EXISTS pstampFile; DROP TABLE IF EXISTS staticskyInput; DROP TABLE IF EXISTS staticskyResult; @@ -97,4 +98,14 @@ DROP TABLE IF EXISTS diffPhotRun; DROP TABLE IF EXISTS diffPhotSkyfile; +DROP TABLE IF EXISTS lapExp; +DROP TABLE IF EXISTS lapRun; +DROP TABLE IF EXISTS lapSequence; +DROP TABLE IF EXISTS minidvodbRun; +DROP TABLE IF EXISTS minidvodbCopy; +DROP TABLE IF EXISTS minidvodbProcessed; +DROP TABLE IF EXISTS vpProcessedCell; +DROP TABLE IF EXISTS vpRun; +DROP TABLE IF EXISTS skycalRun; +DROP TABLE IF EXISTS skycalResult; SET FOREIGN_KEY_CHECKS=1 Index: branches/meh_branches/ppstack_test/ippTools/share/pztool_pendingimfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/pztool_pendingimfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/pztool_pendingimfile.sql (revision 33415) @@ -19,3 +19,24 @@ AND pzDownloadImfile.class IS NULL AND pzDownloadImfile.class_id IS NULL + AND summitImfile.exp_name NOT LIKE '%a' +UNION + SELECT + summitImfile.*, + summitExp.dateobs + FROM summitImfile + JOIN pzDownloadExp + USING(summit_id) + LEFT JOIN pzDownloadImfile + USING(summit_id, class, class_id) + JOIN summitExp + USING(summit_id) + WHERE + pzDownloadExp.state = 'run' + AND pzDownloadImfile.summit_id IS NULL + AND pzDownloadImfile.exp_name IS NULL + AND pzDownloadImfile.camera IS NULL + AND pzDownloadImfile.telescope IS NULL + AND pzDownloadImfile.class IS NULL + AND pzDownloadImfile.class_id IS NULL + AND summitImfile.exp_name LIKE '%a' ) as Foo Index: branches/meh_branches/ppstack_test/ippTools/share/stacktool_inputskyfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/stacktool_inputskyfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/stacktool_inputskyfile.sql (revision 33415) @@ -1,5 +1,10 @@ SELECT warpSkyfile.*, - rawExp.camera + rawExp.camera, + rawExp.exp_id, + rawExp.exp_name, + rawExp.object, + rawExp.dateobs, + rawExp.comment FROM stackRun JOIN stackInputSkyfile USING(stack_id) Index: branches/meh_branches/ppstack_test/ippTools/share/staticskytool_defineskycalrun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/staticskytool_defineskycalrun.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/staticskytool_defineskycalrun.sql (revision 33415) @@ -0,0 +1,16 @@ +SELECT + sky_id, + stack_id, + staticskyRun.workdir, + staticskyRun.label, + staticskyRun.data_group, + staticskyRun.dist_group +FROM staticskyRun + JOIN staticskyResult USING(sky_id) + JOIN staticskyInput USING(sky_id) + JOIN stackRun USING(stack_id) + JOIN stackSumSkyfile USING(stack_id) + LEFT JOIN skycalRun USING(sky_id, stack_id) +WHERE staticskyRun.state = 'full' + AND staticskyResult.quality = 0 + Index: branches/meh_branches/ppstack_test/ippTools/share/staticskytool_pendingskycalrun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/staticskytool_pendingskycalrun.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/staticskytool_pendingskycalrun.sql (revision 33415) @@ -0,0 +1,21 @@ +SELECT + skycalRun.*, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + staticskyResult.path_base, + rawExp.camera, + staticskyResult.num_inputs = 1 AS singlefilter +FROM skycalRun +JOIN staticskyRun USING (sky_id) +JOIN staticskyResult USING(sky_id) +JOIN stackRun USING (stack_id) +JOIN stackInputSkyfile USING(stack_id) +JOIN warpRun USING(warp_id) +JOIN fakeRun USING(fake_id) +JOIN camRun USING(cam_id) +JOIN chipRun USING (chip_id) +JOIN rawExp USING(exp_id) +LEFT JOIN skycalResult USING(skycal_id) +WHERE + (skycalRun.state = 'new' AND skycalResult.skycal_id IS NULL) Index: branches/meh_branches/ppstack_test/ippTools/share/staticskytool_result.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/staticskytool_result.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/staticskytool_result.sql (revision 33415) @@ -1,3 +1,3 @@ -SELECT DISTINCT +SELECT staticskyResult.*, staticskyRun.state, @@ -6,5 +6,5 @@ stackRun.tess_id, stackRun.skycell_id, - stackRun.filter + count(stackRun.filter) AS num_filters FROM staticskyRun JOIN staticskyResult USING(sky_id) Index: branches/meh_branches/ppstack_test/ippTools/share/staticskytool_revertskycal.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/staticskytool_revertskycal.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/staticskytool_revertskycal.sql (revision 33415) @@ -0,0 +1,6 @@ +DELETE FROM skycalResult +USING skycalResult, skycalRun, stackRun +WHERE skycalRun.skycal_id = skycalResult.skycal_id + AND stackRun.stack_id = skycalRun.stack_id + AND skycalRun.state = 'new' + AND skycalResult.fault != 0 Index: branches/meh_branches/ppstack_test/ippTools/share/staticskytool_skycalresult.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/staticskytool_skycalresult.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/staticskytool_skycalresult.sql (revision 33415) @@ -0,0 +1,14 @@ +SELECT + skycalResult.*, + skycalRun.stack_id, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter, + skycalRun.state, + skycalRun.label, + skycalRun.data_group, + skycalRun.sky_id +FROM skycalRun +JOIN skycalResult USING(skycal_id) +JOIN stackRun USING(stack_id) + Index: branches/meh_branches/ppstack_test/ippTools/share/staticskytool_todo.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/staticskytool_todo.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/staticskytool_todo.sql (revision 33415) @@ -6,12 +6,17 @@ staticskyRun.state, stackRun.tess_id, - stackRun.skycell_id + stackRun.skycell_id, + IFNULL(Label.priority, 10000) AS priority FROM staticskyRun JOIN staticskyInput USING (sky_id) JOIN stackRun USING (stack_id) LEFT JOIN staticskyResult USING(sky_id) +LEFT JOIN Label ON staticskyRun.label = Label.label WHERE (staticskyRun.state = 'new' AND staticskyResult.sky_id IS NULL) + AND (Label.active OR Label.active IS NULL) + -- WHERE hook %s GROUP BY sky_id HAVING SUM(IF(stackRun.state = 'full', 1, 0)) = COUNT(staticskyInput.sky_id) +ORDER BY priority DESC, sky_id Index: branches/meh_branches/ppstack_test/ippTools/share/vptool_find_rawexp.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/vptool_find_rawexp.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/vptool_find_rawexp.sql (revision 33415) @@ -0,0 +1,12 @@ +-- this query is used to find potental rawExps to be queued for vptool +-- processeing +SELECT DISTINCT + rawExp.*, + newExp.label +FROM rawExp +JOIN newExp using (exp_id) +JOIN rawImfile using(exp_id) +LEFT JOIN vpRun USING (exp_id) +WHERE + rawExp.fault = 0 + AND rawImfile.video_cells Index: branches/meh_branches/ppstack_test/ippTools/share/vptool_pendingimfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/vptool_pendingimfile.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/vptool_pendingimfile.sql (revision 33415) @@ -0,0 +1,12 @@ +SELECT + vpRun.*, + rawImfile.class_id, + rawImfile.uri +FROM vpRun + JOIN rawExp USING(exp_id) + JOIN rawImfile USING(exp_id) + LEFT JOIN vpProcessedCell USING(vp_id, class_id) +WHERE rawImfile.video_cells + AND vpRun.state = 'new' + AND vpProcessedCell.vp_id IS NULL + Index: branches/meh_branches/ppstack_test/ippTools/share/vptool_pendingrun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/vptool_pendingrun.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/vptool_pendingrun.sql (revision 33415) @@ -0,0 +1,17 @@ +SELECT vp_id, + rawExp.camera, + CONCAT_WS('.', exp_name, exp_id) AS exp_tag, + vpRun.label, + vpRun.workdir, + vpRun.dest_id, + rcDestination.name AS dest_name, + rcDestination.dbname AS ds_dbname, + rcDestination.dbhost AS ds_dbhost, + IFNULL(Label.priority, 10000) as priority +FROM vpRun + JOIN rawExp USING(exp_id) + LEFT JOIN rcDestination USING(dest_id) + LEFT JOIN Label ON vpRun.label = Label.label +WHERE vpRun.state = 'new' + AND vpRun.fault = 0 + AND (Label.active OR Label.active IS NULL) Index: branches/meh_branches/ppstack_test/ippTools/share/vptool_processedcell.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/vptool_processedcell.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/vptool_processedcell.sql (revision 33415) @@ -0,0 +1,22 @@ +SELECT + vpProcessedCell.*, + vpRun.state, + vpRun.label, + vpRun.data_group, + vpRun.outroot, + rawExp.exp_id, + rawExp.exp_tag, + rawExp.exp_name, + rawExp.filter, + rawExp.camera, + rawExp.telescope, + rawExp.filelevel, + rawExp.dateobs, + rawExp.obs_mode, + rawExp.comment, + rawExp.ra, + rawExp.decl, + rawExp.exp_time +FROM vpRun + JOIN vpProcessedCell USING(vp_id) + JOIN rawExp USING(exp_id) Index: branches/meh_branches/ppstack_test/ippTools/share/vptool_revertrun.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/vptool_revertrun.sql (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/share/vptool_revertrun.sql (revision 33415) @@ -0,0 +1,4 @@ +UPDATE vpRun + SET fault = 0 +WHERE vpRun.state = 'new' + AND fault > 0 Index: branches/meh_branches/ppstack_test/ippTools/share/warptool_change_run_state.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/warptool_change_run_state.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/warptool_change_run_state.sql (revision 33415) @@ -17,3 +17,4 @@ warpSkyfile.warp_id = warpRun.warp_id AND warpSkyfile.data_state != '%s' + AND warpSkyfile.quality = 0 ) = 0 Index: branches/meh_branches/ppstack_test/ippTools/share/warptool_pendingcleanupskyfile.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 33415) @@ -11,5 +11,6 @@ WHERE ((warpRun.state = 'goto_cleaned' AND (warpSkyfile.data_state = 'full' - OR warpSkyfile.data_state = 'update')) + OR warpSkyfile.data_state = 'update') + AND warpSkyfile.quality = 0) OR (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed') Index: branches/meh_branches/ppstack_test/ippTools/share/warptool_scmap.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/warptool_scmap.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/warptool_scmap.sql (revision 33415) @@ -29,4 +29,5 @@ ON chipRun.chip_id = chipProcessedImfile.chip_id AND warpSkyCellMap.class_id = chipProcessedImfile.class_id + AND chipProcessedImfile.quality = 0 JOIN rawImfile ON chipRun.exp_id = rawImfile.exp_id Index: branches/meh_branches/ppstack_test/ippTools/share/warptool_setskyfiletoupdate.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/warptool_setskyfiletoupdate.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/warptool_setskyfiletoupdate.sql (revision 33415) @@ -8,2 +8,3 @@ AND (warpRun.state = 'cleaned' OR warpRun.state = 'update') AND (warpSkyfile.data_state = 'cleaned' OR warpSkyfile.data_state = 'update') + AND (warpSkyfile.quality = 0) Index: branches/meh_branches/ppstack_test/ippTools/share/warptool_towarped.sql =================================================================== --- branches/meh_branches/ppstack_test/ippTools/share/warptool_towarped.sql (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/share/warptool_towarped.sql (revision 33415) @@ -100,10 +100,11 @@ AND warpSkyfile.fault = 0 AND camRun.state = 'full' + AND chipProcessedImfile.quality = 0 + -- Maybe add this to solve the if any chip is bad a whole warp can't be updated problem? + -- AND (chipProcessedImfile.fault = 0 OR chipProcessedImfile.fault = 26) AND (Label.active OR Label.active IS NULL) -- where hook 2 %s GROUP BY warp_id, skycell_id - -- if warpSkyfile was magicked previously require inputs to be magicked - -- this blocks processing until all the chip inputs have been destreaked - HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full' and (chipRun.magicked = 0 OR chipProcessedImfile.magicked > 0), 1, 0)) + HAVING COUNT(warpSkyCellMap.class_id) = SUM(IF(chipProcessedImfile.data_state ='full', 1, 0)) -- limit hook 2 %s ) Index: branches/meh_branches/ppstack_test/ippTools/src/Makefile.am =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/Makefile.am (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/Makefile.am (revision 33415) @@ -29,5 +29,7 @@ diffphottool \ minidvodbtool \ - laptool + mergetool \ + laptool \ + vptool pkginclude_HEADERS = \ @@ -42,4 +44,5 @@ pxmagic.h \ pxminidvodb.h \ + pxmerge.h \ pxregister.h \ pxspace.h \ @@ -78,5 +81,7 @@ diffphottool.h \ minidvodbtool.h \ - laptool.h + mergetool.h \ + laptool.h \ + vptool.h lib_LTLIBRARIES = libpxtools.la @@ -96,4 +101,5 @@ pxmagic.c \ pxminidvodb.c \ + pxmerge.c \ pxregister.c \ pxspace.c \ @@ -292,4 +298,10 @@ minidvodbtoolConfig.c +mergetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +mergetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +mergetool_SOURCES = \ + mergetool.c \ + mergetoolConfig.c + laptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) laptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la @@ -298,4 +310,10 @@ laptoolConfig.c +vptool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +vptool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +vptool_SOURCES = \ + vptool.c \ + vptoolConfig.c + clean-local: -rm -f TAGS Index: branches/meh_branches/ppstack_test/ippTools/src/addtool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/addtool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/addtool.c (revision 33415) @@ -143,5 +143,11 @@ pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label } - + if (strcmp(stage, "staticsky_multi")== 0) { + + pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); + //no filter here + } if (!psListLength(where->list)) { psFree(where); @@ -237,5 +243,19 @@ } - + if (strcmp(stage,"staticsky_multi") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); + } + } if (!bare_query) { @@ -272,7 +292,8 @@ } else { //This picks only the unmagicked/uncensored ones - if (strcmp(stage,"cam") == 0) { - psStringAppend(&query, " AND (camRun.magicked = 0)"); - } + //if (strcmp(stage,"cam") == 0) { + //we can now properly handle the magicked case in ippScripts, so we queue camRuns in any magic state now if -uncensored. + //psStringAppend(&query, " AND (camRun.magicked = 0)"); + //} if (strcmp(stage,"stack") == 0) { psStringAppend(&query, " AND (stackRun.magicked = 0)"); @@ -280,4 +301,20 @@ } + + //if we grab a group of camRun/stackRuns that have multiple camRun/stackRun pointing to the same exp_id/stack_id, and that exp_id/stack_id has never before been added to addRun, we need to group by exp_id/stack_id to ensure that only 1 of the camRun/stackRun for that exp_id is added to the addRun stage. + + if (strcmp(stage,"cam") == 0) { + psStringAppend(&query, " GROUP BY exp_id"); + } + if (strcmp(stage,"stack") == 0) { + psStringAppend(&query, " GROUP BY stack_id"); + } + if (strcmp(stage,"staticsky") == 0) { + psStringAppend(&query, " GROUP BY stack_id"); + } + if (strcmp(stage,"staticsky_multi") == 0) { + psStringAppend(&query, " GROUP BY sky_id"); //some reason it needs this + } + psTrace("addtool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); @@ -391,4 +428,43 @@ } } + if (strcmp(stage,"staticsky_multi") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + if (num_inputs < 0) { + psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs"); + return false; + } + + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb) { //there's no staticsky.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + + psFree(row); + } + } + + + // start a transaction so we don't end up with an exp without any associted @@ -424,4 +500,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -465,4 +542,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -506,4 +584,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -530,4 +609,57 @@ } + if (strcmp(stage,"staticsky_multi") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + stage_id = row->sky_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + for (int filter_id = 0; filter_id < num_inputs; filter_id++) { + if (!pxaddQueueByCamID(config, + stage, + stage_id, + filter_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + } + psFree(row); + } + } + + + + psFree(output); @@ -571,4 +703,7 @@ } if (strcmp(stage, "staticsky")==0) { + query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); + } + if (strcmp(stage, "staticsky_multi")==0) { query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); } @@ -612,5 +747,8 @@ query = pxDataGet("addtool_find_pendingexp_staticsky.sql"); } - + if (strcmp(stage, "staticsky_multi")==0) { + query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql"); + } + if (!query) { @@ -634,5 +772,8 @@ psStringAppend(&query, " GROUP BY %s", "sky_id"); } - + if (strcmp(stage, "staticsky_multi") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); + } // treat limit == 0 as "no limit" if (limit) { @@ -684,5 +825,5 @@ PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-minidvodb_name", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - + PXOPT_LOOKUP_S64(stage_extra1, config->args, "-stage_extra1", false, false); // generate restrictions psMetadata *where = psMetadataAlloc(); @@ -762,5 +903,19 @@ } } - + //if there is a stage_extra1, set it in addRun (it's not known until it is processed) + if (stage_extra1) { + psString setExtra = NULL; + psStringAppend (&setExtra, "UPDATE addRun set stage_extra1 = %" PRId64, stage_extra1); + psStringAppend (&setExtra, " where add_id = %" PRId64, row->add_id); + + if (!p_psDBRunQuery(config->dbh, setExtra)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + + return false; + } + } @@ -824,4 +979,8 @@ query = pxDataGet("addtool_find_processedexp_staticsky.sql"); } + if (strcmp (stage,"staticsky_multi") == 0) { + query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql"); + } + if (!query) { @@ -905,5 +1064,5 @@ PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); PXOPT_COPY_S64(config->args, where, "-stage_id", "addRun.stage_id", "=="); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); @@ -934,4 +1093,10 @@ query = pxDataGet("addtool_revertprocessedexp_staticsky.sql"); } + if (strcmp(stage, "staticsky_multi") == 0) { + query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql"); + } + + + if (!query) { // rollback @@ -1071,7 +1236,5 @@ PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", true, false); - - //optional + //optional PXOPT_LOOKUP_STR(minidvodb_name, config->args, "-set_minidvodb_name", false, false); PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); @@ -1088,4 +1251,7 @@ if (minidvodb_path) { minidvodbpath = minidvodb_path; + } else { + psError(PS_ERR_UNKNOWN, false, "require minidvodb_path"); + return false; } @@ -1095,6 +1261,5 @@ minidvodb_group, minidvodbpath, - mergedvodb_path, - "new", + "new", 0 )) { @@ -1118,9 +1283,4 @@ psStringAppend(&minidvodb_name, "%s.%" PRIu64,minidvodb_group,minidvodb_id); } - - if (!minidvodb_path) { - psStringAppend(&minidvodb_path,"%s/%s",mergedvodb_path,minidvodb_name); - } - if (minidvodb_path) { psStringAppend(&minidvodb_path,"/%s",minidvodb_name); @@ -1154,5 +1314,4 @@ PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_path", "minidvodb_path", "=="); - PXOPT_COPY_STR(config->args, where, "-mergedvodb_path", "mergedvodb_path", "=="); PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodb_group", "=="); @@ -1160,5 +1319,4 @@ PXOPT_LOOKUP_STR(minidvodb_path, config->args, "-set_minidvodb_path", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-set_minidvodb_group", false, false); @@ -1195,12 +1353,4 @@ } - if (mergedvodb_path) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); - cnt++; - } - if (minidvodb_group) { if (cnt) { @@ -1246,5 +1396,5 @@ } - psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL"); + psString firstquery = psStringCopy("SELECT * from minidvodbRun where state = 'new' and minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL"); psString firstwhereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -1294,5 +1444,5 @@ //now flip new -> active - psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL and mergedvodb_path IS NOT NULL "); + psString query2 = psStringCopy("UPDATE minidvodbRun SET state = 'active' WHERE state = 'new' AND minidvodb_name is NOT NULL and minidvodb_group IS NOT NULL and minidvodb_path IS NOT NULL "); psStringAppend(&query2, " AND minidvodb_group = '%s' limit 1;", minidvodb_group); if (!p_psDBRunQuery(config->dbh, query2)) { @@ -1471,13 +1621,9 @@ // required PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-mergedvodb_path", true, false); PXOPT_LOOKUP_STR(minidvodb_group, config->args, "-minidvodb_group", true, false); // optional - PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); - PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); @@ -1530,12 +1676,8 @@ minidvodbProcessedRow *row = minidvodbProcessedRowAlloc( pendingRow->minidvodb_id, - merge_order, dtime_resort, dtime_relphot, - dtime_verify, - dtime_merge, dtime_script, epoch, - mergedvodb_path, fault ); @@ -1552,67 +1694,4 @@ } - - - - //this finds the # of merged things (for the merge order) - psString query3 = NULL; - psStringAppend(&query3, "select count(*) from minidvodbRun join minidvodbProcessed using (minidvodb_id) where state = 'merged' and minidvodb_group = '%s';", minidvodb_group); - - if (!p_psDBRunQuery(config->dbh, query3)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query3); - return false; - } - psArray *output2 = p_psDBFetchResult(config->dbh); - if (!output2) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output2)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - bool status; - psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); - if (!status) { - - psAbort("failed to lookup value for count column"); - return false; - } - psString final = NULL; - psStringAppend(&final, "%" PRIu64, m_order); - //return false; - psFree(query3); - psFree(output2); - - - - //update the merge_order - - psString query4 = NULL; - psStringAppend(&query4, "update minidvodbProcessed set merge_order = %"PRIu64,m_order); - psStringAppend(&query4," where minidvodb_id = %" PRIu64, minidvodb_id); - //printf("%s", query4); - if (!p_psDBRunQuery(config->dbh, query4)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query4); - return false; - } - // -psFree(query4); - - - -// since there is only one exp per 'new' set mindvodbRun.state = 'merged' - psString query2 = NULL ; psStringAppend(&query2, "UPDATE minidvodbRun SET state = 'merged' WHERE minidvodb_id = %'" PRIu64, row->minidvodb_id); @@ -1627,12 +1706,4 @@ return false; } - - - - - - - - psFree(row); psFree(pendingRow); @@ -1647,10 +1718,4 @@ return false; } - - - //print the merge_order (why not!) - printf("%s", final); - psFree(final); - return true; } @@ -1661,6 +1726,6 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); - PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbProcessed.minidvodb_name", "=="); - PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbProcessed.minidvodb_group", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_name", "minidvodbRun.minidvodb_name", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -1811,10 +1876,7 @@ PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-minidvodb_id", true, false); - PXOPT_LOOKUP_U64(merge_order, config->args, "-set_merge_order", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); PXOPT_LOOKUP_F32(dtime_relphot, config->args, "-set_dtime_relphot", false, false); PXOPT_LOOKUP_F32(dtime_resort, config->args, "-set_dtime_resort", false, false); - PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); -PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbProcessed.minidvodb_id", "=="); @@ -1836,13 +1898,4 @@ } - if (merge_order) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - - psStringAppend(&query, " merge_order = %" PRId64, merge_order); - cnt++; - } - if (dtime_relphot) { if (cnt) { @@ -1861,19 +1914,5 @@ } - if (dtime_merge) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_merge = %f", dtime_merge); - cnt++; - } -if (dtime_verify) { - if (cnt) { - psStringAppend(&query, "%s", comma); - } - psStringAppend(&query, " dtime_verify = %f", dtime_verify); - cnt++; - } -if (dtime_script) { + if (dtime_script) { if (cnt) { psStringAppend(&query, "%s", comma); Index: branches/meh_branches/ppstack_test/ippTools/src/addtoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/addtoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/addtoolConfig.c (revision 33415) @@ -115,4 +115,6 @@ psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); +psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-stage_extra1", 0, "set stage_extra1", 0); + // -processedexp @@ -165,5 +167,4 @@ psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define minidvodb_group (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for the merged dvodb (required)", NULL); psMetadataAddStr(addminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -174,9 +175,7 @@ psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name (LIKE)", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-minidvodb_path", 0, "search by path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-mergedvodb_path", 0, "search by path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_name", 0, "define minidvodb_name", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_path", 0, "define path for minidvodb", NULL); - psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define path for minidvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_minidvodb_group", 0, "define path for the merged dvodb", NULL); psMetadataAddStr(updateminidvodbrunArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -205,12 +204,8 @@ psMetadata *addminidvodbprocessedArgs = psMetadataAlloc(); psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "define minidvodb_id (required)", 0); - psMetadataAddU64(addminidvodbprocessedArgs, PS_LIST_TAIL, "-merge_order", 0, "define merge order", 0); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_relphot", 0, "define elapsed time for relphot (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_resort", 0, "define elapsed time for resort (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); - psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); psMetadataAddF32(addminidvodbprocessedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); psMetadataAddTime(addminidvodbprocessedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); - psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-mergedvodb_path",0, "path of merged dvodb", NULL); psMetadataAddStr(addminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_group",0, "minidvodb_group", NULL); psMetadataAddS16(addminidvodbprocessedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); @@ -232,9 +227,6 @@ psMetadataAddStr(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-minidvodb_name", 0, "search by minidvodb_name", NULL); psMetadataAddS16(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); - psMetadataAddU64(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_merge_order", 0, "define merge order", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_relphot", 0, "define elapsed time for relphot (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_resort", 0, "define elapsed time for resort (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); - psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); psMetadataAddF32(updateminidvodbprocessedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); psMetadata *argSets = psMetadataAlloc(); Index: branches/meh_branches/ppstack_test/ippTools/src/bgtoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/bgtoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/bgtoolConfig.c (revision 33415) @@ -113,4 +113,6 @@ psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL); psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); + psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-not_destreaked", 0, "define magicked status", 0); psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); @@ -298,4 +300,6 @@ psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL); psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for runs that have been destreaked", false); + psMetadataAddS64(updatewarpArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked status", 0); psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false); psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); Index: branches/meh_branches/ppstack_test/ippTools/src/camtool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/camtool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/camtool.c (revision 33415) @@ -112,4 +112,5 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "=="); if (!psListLength(where->list) && Index: branches/meh_branches/ppstack_test/ippTools/src/camtoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/camtoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/camtoolConfig.c (revision 33415) @@ -55,4 +55,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-data_group", 0, "search by chipRun data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by rawExp obs_mode", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); Index: branches/meh_branches/ppstack_test/ippTools/src/chiptool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/chiptool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/chiptool.c (revision 33415) @@ -43,4 +43,5 @@ static bool revertprocessedimfileMode(pxConfig *config); static bool updateprocessedimfileMode(pxConfig *config); +static bool dropprocessedimfileMode(pxConfig *config); static bool advanceexpMode(pxConfig *config); static bool blockMode(pxConfig *config); @@ -88,4 +89,5 @@ MODECASE(CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileMode); MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode); + MODECASE(CHIPTOOL_MODE_DROPPROCESSEDIMFILE, dropprocessedimfileMode); MODECASE(CHIPTOOL_MODE_ADVANCEEXP, advanceexpMode); MODECASE(CHIPTOOL_MODE_BLOCK, blockMode); @@ -196,4 +198,11 @@ GET_VALUE (psS64, S64, exp_id, "exp_id"); GET_VALUE (psString, Str, raw_workdir, "workdir"); +#ifdef notyet + GET_VALUE (psString, Str, old_workdir, "old_workdir"); + GET_VALUE (psString, Str, old_data_group, "old_data_group"); +#else + const char *old_workdir = NULL; + const char *old_data_group = NULL; +#endif GET_VALUE (psString, Str, raw_label, "label"); GET_VALUE (psString, Str, raw_reduction, "reduction"); @@ -211,7 +220,7 @@ if (!pxchipQueueByExpTag(config, exp_id, - workdir ? workdir : raw_workdir, + workdir ? workdir : (old_workdir ? old_workdir : raw_workdir), label ? label : raw_label, - data_group ? data_group : (label ? label : raw_label), + data_group ? data_group : (old_data_group ? old_data_group : (label ? label : raw_label)), dist_group, reduction ? reduction : raw_reduction, @@ -343,4 +352,5 @@ pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "chipOld.state", "=="); // psListLength(where->list) is at least 1 because exp_type defaults to "object" @@ -352,5 +362,5 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); @@ -466,4 +476,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "=="); psString query = pxDataGet("chiptool_pendingimfile.sql"); @@ -883,4 +894,6 @@ } psFree(query); + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("chiptool", PS_LOG_INFO, "Deleted %d chipProcessedImfiles", numDeleted); if (!fault) { @@ -896,4 +909,7 @@ } psFree(query_update); + int numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("chiptool", PS_LOG_INFO, "Updated %d chipProcessedImfiles", numUpdated); + return true; @@ -962,5 +978,5 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); if (psListLength(where->list) == 0) { psError(PS_ERR_UNKNOWN, true, "search parameters are required"); @@ -986,4 +1002,76 @@ + return true; +} + +// "drop" previously processed components as though the original processing +// had poor quality. Sets quality, data_state, clears fault and magicked +// Deletes corresponding magicDSFile (if one exists) +static bool dropprocessedimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipProcessedImfile.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", true, false); + + if (!quality) { + psError(PXTOOLS_ERR_CONFIG, true, "non-zero quality value is required"); + return false; + } + + if (psListLength(where->list) == 0) { + // won't get here + psError(PXTOOLS_ERR_CONFIG, true, "-chip_id and -class_id are required"); + return false; + } + + psString query = pxDataGet("chiptool_dropprocessedimfile_update.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString delete_query = pxDataGet("chiptool_dropprocessedimfile_delete.sql"); + if (!delete_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(where); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, quality)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error on rollback"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psStringAppend(&delete_query, " AND %s", whereClause); + psFree(whereClause); + if (!p_psDBRunQuery(config->dbh, delete_query)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error on rollback"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete_query); + return false; + } + psFree(delete_query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; } @@ -1418,5 +1506,5 @@ psMetadata *where = psMetadataAlloc(); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); // look for completed chipPendingExp // migrate them to chipProccessedExp & camPendingExp @@ -1432,4 +1520,6 @@ psFree(whereClause); } + + psTrace("chiptool", 10, "%s\n", query); psFree(where); @@ -1943,7 +2033,16 @@ return false; } - psFree(setHook); psFree(query); + + // if we are updating a whole chipRun set lingering chips with poor quality to full data_state + if (!class_id) { + query = "UPDATE chipProcessedImfile SET data_state ='full', fault = 0 WHERE chip_id = %" PRId64 " AND quality != 0 AND (data_state ='cleaned' OR data_state = 'update')"; + + if (!p_psDBRunQueryF(config->dbh, query, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } return true; Index: branches/meh_branches/ppstack_test/ippTools/src/chiptool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/chiptool.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/chiptool.h (revision 33415) @@ -33,4 +33,5 @@ CHIPTOOL_MODE_REVERTPROCESSEDIMFILE, CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE, + CHIPTOOL_MODE_DROPPROCESSEDIMFILE, CHIPTOOL_MODE_SETIMFILETOUPDATE, CHIPTOOL_MODE_ADVANCEEXP, Index: branches/meh_branches/ppstack_test/ippTools/src/chiptoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/chiptoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/chiptoolConfig.c (revision 33415) @@ -72,6 +72,7 @@ psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object"); - - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-state", 0, "search by chipRun.state", NULL); + + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); @@ -111,4 +112,5 @@ psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0); pxchipSetSearchArgs(pendingimfileArgs); psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -237,5 +239,12 @@ psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); - // -processedExp + + // -dropprocessedimfile + psMetadata *dropprocessedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(dropprocessedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID (required)", 0); + psMetadataAddStr(dropprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID (required)", NULL); + psMetadataAddS16(dropprocessedimfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality (required)", 0); + + // -listrun psMetadata *listrunArgs = psMetadataAlloc(); pxchipSetSearchArgs(listrunArgs); @@ -372,4 +381,5 @@ PXOPT_ADD_MODE("-processedimfile", "show processed imfiles", CHIPTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs); PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); + PXOPT_ADD_MODE("-dropprocessedimfile","drop previously procesed imfile", CHIPTOOL_MODE_DROPPROCESSEDIMFILE, dropprocessedimfileArgs); PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); Index: branches/meh_branches/ppstack_test/ippTools/src/difftool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/difftool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/difftool.c (revision 33415) @@ -1634,4 +1634,6 @@ PXOPT_COPY_F32(config->args, selectWhere, "-timediff", "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<="); + PXOPT_COPY_F32(config->args, selectWhere, "-mintimediff", + "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", ">="); // other where restrictions: @@ -1867,5 +1869,5 @@ const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id"); psString input_data_group = psMetadataLookupStr(NULL, row, "input_data_group"); - if (input_data_group) { + if (!data_group) { data_group = input_data_group; } @@ -1934,19 +1936,70 @@ return false; } + psS64 numSkycells = psDBAffectedRows(config->dbh); + if (numSkycells > 0) { + if (!setdiffRunState(config, run->diff_id, "new", false)) { + psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64, + run->diff_id); + psFree(input); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + run->state = psStringCopy("new"); + } else { + // No overlap between the warps. Insert a dummy skycell which will complete this pair of warps + // XXX: we'd like to use diffInputSkyfileRowAlloc and diffInputSkyfileInsert but there doesn't + // seem to be a way to pass in NULL for skycell_id, stack1 and stack2 + psString dummyQuery = NULL; + psStringAppend(&dummyQuery, + "INSERT INTO diffInputSkyfile VALUES(%" PRId64 ", NULL, %s, NULL, %s, NULL, '%s', 0)", + run->diff_id, + input, // warp1 + template, // warp2 + tess_id); + if (!p_psDBRunQuery(config->dbh, dummyQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(dummyQuery); + psFree(input); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(dummyQuery); + psString finishQuery = NULL; + psStringAppend( &finishQuery, + "UPDATE diffRun set state ='full', dist_group = NULL, note = 'dummy run - no overlap' WHERE diff_id = %" PRId64, + run->diff_id); + if (!p_psDBRunQuery(config->dbh, finishQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(finishQuery); + psFree(input); + psFree(diff); + psFree(run); + psFree(list); + psFree(insert); + psFree(results); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(finishQuery); + run->state = psStringCopy("full"); + } psFree(input); psFree(diff); - - if (!setdiffRunState(config, run->diff_id, "new", false)) { - psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64, - run->diff_id); - psFree(run); - psFree(list); - psFree(insert); - psFree(results); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } psArrayAdd(list, list->n, run); @@ -2461,5 +2514,10 @@ psString limitString = psDBGenerateLimitSQL(limit); psStringAppend(&query, " %s", limitString); + psStringSubstitute(&query,limitString,"@INNERLIMITS@"); psFree(limitString); + + } + else { + psStringSubstitute(&query,NULL,"@INNERLIMITS@"); } // fprintf(stderr,"%s",query); Index: branches/meh_branches/ppstack_test/ippTools/src/difftoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/difftoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/difftoolConfig.c (revision 33415) @@ -283,4 +283,5 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-timediff", 0, "limit time difference between input and template", NAN); + psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-mintimediff", 0, "limit time difference between input and template to be greater than this", NAN); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-backwards", 0, "Template comes after input?", false); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-rotdiff", 0, "limit rotator difference between input and template", NAN); Index: branches/meh_branches/ppstack_test/ippTools/src/disttool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/disttool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/disttool.c (revision 33415) @@ -34,7 +34,9 @@ static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); -static bool rerunMode(pxConfig *config); +static bool startoverMode(pxConfig *config); static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool updateprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); @@ -84,7 +86,8 @@ MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); - MODECASE(DISTTOOL_MODE_RERUN, rerunMode); + MODECASE(DISTTOOL_MODE_STARTOVER, startoverMode); MODECASE(DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentMode); MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); + MODECASE(DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); @@ -141,5 +144,4 @@ PXOPT_LOOKUP_BOOL(use_alternate, config->args, "-use_alternate", false); PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); - PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); @@ -165,10 +167,7 @@ PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); - // queuing no_magic runs can be quite dangerous. Require confirmation - if (no_magic && !pretend && !i_am_sure) { - psError(PXTOOLS_ERR_SYS, true, "-i_am_sure required to queue no_magic distribution runs"); - return false; - } - + PXOPT_LOOKUP_BOOL(single, config->args, "-singlefilter", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + if (use_alternate) { if (strcmp(stage, "raw")) { @@ -361,5 +360,9 @@ } else if (!strcmp(stage, "sky")) { magicRunType = "staticskyRun"; - query = pxDataGet("disttool_definebyquery_sky.sql"); + if (single) { + query = pxDataGet("disttool_definebyquery_sky_singlefilter.sql"); + } else { + query = pxDataGet("disttool_definebyquery_sky.sql"); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -406,4 +409,7 @@ } } + if (!rerun) { + psStringAppend(&query, " AND (distRun.dist_id IS NULL)"); + } if (psListLength(where->list)) { @@ -418,4 +424,12 @@ if (!no_magic) { psStringAppend(&query, " AND (%s.magicked > 0)", magicRunType); + } + + if (!strcmp(stage, "sky")) { + if (single) { + psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1"); + } else { + psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) > 1"); + } } @@ -636,5 +650,5 @@ } -static bool rerunMode(pxConfig *config) +static bool startoverMode(pxConfig *config) { psMetadata *where = psMetadataAlloc(); @@ -646,5 +660,5 @@ // require data_group or dist_id to be supplied - PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", true, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", false, false); if (!data_group && !dist_id) { @@ -771,5 +785,5 @@ PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + pxAddLabelSearchArgs(config, where, "-label", "distRun.label", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); @@ -962,4 +976,60 @@ return true; } +static bool updateprocessedcomponentMode(pxConfig *config) +{ + + // required values + PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); + PXOPT_LOOKUP_STR(component, config->args, "-component", true, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); + PXOPT_LOOKUP_STR(outdir, config->args, "-outdir", false, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + + bool setfault = clearfault || fault; + + if (!bytes && !md5sum && !outdir && !name && !setfault && !fault) { + psError(PS_ERR_UNKNOWN, true, "at least one of bytes md5sum outdir name fault or setfault is required"); + return false; + } + + char *sep = ""; + psString query = psStringCopy("UPDATE distComponent SET "); + if (setfault) { + psStringAppend(&query, "%s fault = %d", sep, fault ? 1 : 0); + sep = ", "; + } + if (bytes) { + psStringAppend(&query, "%s bytes = %d", sep, bytes); + sep = ", "; + } + if (md5sum) { + psStringAppend(&query, "%s md5sum = '%s'", sep, md5sum); + sep = ", "; + } + if (outdir) { + psStringAppend(&query, "%s outdir = '%s'", sep, outdir); + sep = ", "; + } + if (name) { + psStringAppend(&query, "%s name = '%s'", sep, name); + sep = ", "; + } + + psStringAppend(&query, "\nWHERE dist_id = %"PRId64 " AND component = '%s'", + dist_id, component); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} static bool toadvanceMode(pxConfig *config) @@ -1041,4 +1111,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-component", "component", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -2178,4 +2249,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); if (!psListLength(where->list)) { Index: branches/meh_branches/ppstack_test/ippTools/src/disttool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/disttool.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/disttool.h (revision 33415) @@ -28,9 +28,10 @@ DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, - DISTTOOL_MODE_RERUN, + DISTTOOL_MODE_STARTOVER, DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, + DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, DISTTOOL_MODE_PENDINGCLEANUP, Index: branches/meh_branches/ppstack_test/ippTools/src/disttoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/disttoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/disttoolConfig.c (revision 33415) @@ -49,5 +49,4 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirmation needed when -no_magic is supplied", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-use_alternate", 0, "use alternate inputs", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); @@ -64,5 +63,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by run label", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group", 0, "select by dist_group", NULL); - + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-singlefilter", 0, "select single filter runs (sky stage only)", false); + + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue a new distRun even if one already exists", false); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -87,15 +88,15 @@ psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); - // -rerun - psMetadata *rerunArgs = psMetadataAlloc(); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group (required)", NULL); - psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); - psMetadataAddS64(rerunArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); - psMetadataAddStr(rerunArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); - psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); - psMetadataAddU64(rerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(rerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -startover + psMetadata *startoverArgs = psMetadataAlloc(); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-data_group", 0, "define data_group", NULL); + psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-dist_id", 0, "search by dist_id", 0); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-stage", 0, "search by stage", NULL); + psMetadataAddS64(startoverArgs, PS_LIST_TAIL, "-stage_id", 0, "search by stage_id", 0); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(startoverArgs, PS_LIST_TAIL, "-set_label", 0, "define new label", NULL); + psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-pretend", 0, "don't queue runs just display what would be selected", false); + psMetadataAddU64(startoverArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(startoverArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertrun @@ -105,5 +106,5 @@ psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); - psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); psMetadataAddBool(revertrunArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); @@ -151,6 +152,18 @@ psMetadata *processedcomponentArgs = psMetadataAlloc(); psMetadataAddS64(processedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(processedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); psMetadataAddU64(processedcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateprocessedcomponentArgs + psMetadata *updateprocessedcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-outdir", 0, "define output directory", NULL); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); + psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0); + psMetadataAddStr(updateprocessedcomponentArgs, PS_LIST_TAIL, "-md5sum", 0, "define stage for bundle", NULL); + psMetadataAddS32(updateprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(updateprocessedcomponentArgs, PS_LIST_TAIL, "-clearfault", 0, "set fault to zero", false); // -toadvance @@ -180,5 +193,5 @@ psMetadata *updatefilesetArgs = psMetadataAlloc(); psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); -// psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL); psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -343,8 +356,9 @@ PXOPT_ADD_MODE("-definebyquery", "define distribution run", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); PXOPT_ADD_MODE("-updaterun", "update distRun", DISTTOOL_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-rerun", "reprocess a cleaned distribution run", DISTTOOL_MODE_RERUN, rerunArgs); + PXOPT_ADD_MODE("-startover", "reprocess a cleaned distribution run", DISTTOOL_MODE_STARTOVER, startoverArgs); PXOPT_ADD_MODE("-revertrun", "revert distRun.faults", DISTTOOL_MODE_REVERTRUN, revertrunArgs); PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-updateprocessedcomponent", "", DISTTOOL_MODE_UPDATEPROCESSEDCOMPONENT, updateprocessedcomponentArgs); PXOPT_ADD_MODE("-revertcomponent", "revert faulted components", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); Index: branches/meh_branches/ppstack_test/ippTools/src/faketool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/faketool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/faketool.c (revision 33415) @@ -335,4 +335,5 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "fakeRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "=="); PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "=="); @@ -625,9 +626,9 @@ psMetadata *where = psMetadataAlloc(); pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "=="); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp..camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); Index: branches/meh_branches/ppstack_test/ippTools/src/flatcorr.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/flatcorr.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/flatcorr.c (revision 33415) @@ -666,4 +666,5 @@ "cam", row->cam_id, + 0, row->workdir, row->reduction, Index: branches/meh_branches/ppstack_test/ippTools/src/laptool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/laptool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/laptool.c (revision 33415) @@ -30,4 +30,6 @@ static bool updateexpMode(pxConfig *config); +static bool diffcheckMode(pxConfig *config); + static bool inactiveexpMode(pxConfig *config); @@ -62,4 +64,6 @@ MODECASE(LAPTOOL_MODE_UPDATEEXP, updateexpMode); + MODECASE(LAPTOOL_MODE_DIFFCHECK, diffcheckMode); + MODECASE(LAPTOOL_MODE_INACTIVEEXP, inactiveexpMode); default: @@ -276,4 +280,7 @@ PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + psMetadata *chipWhere = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, chipWhere, "-seq_id", "lapRun.seq_id", "=="); + // This seems unnecessarily clunky. if (!all_obsmode) { @@ -296,8 +303,13 @@ } + psString chipWhereClause = psDBGenerateWhereConditionSQL(chipWhere,NULL); if (whereClause) { psStringSubstitute(&query,whereClause,"@WHERE@"); } + if (chipWhereClause) { + psStringSubstitute(&query,chipWhereClause,"@CHIPWHERE@"); + } psFree(where); + psFree(chipWhere); // Fetch exposures @@ -378,4 +390,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); psMetadata *where = psMetadataAlloc(); @@ -399,4 +412,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n %s", limitString); + psFree(limitString); } @@ -755,6 +773,68 @@ } - - +static bool diffcheckMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-lap_id", "lapRun.lap_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-seq_id", "seq_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + + psString query = pxDataGet("laptool_WSdiff_check.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringPrepend(&whereClause, "\n AND "); + psStringSubstitute(&query,whereClause,"@WHERE@"); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error %d",err); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("laptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "lapRunWarpStackSkycells", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return(true); +} static bool inactiveexpMode(pxConfig *config) Index: branches/meh_branches/ppstack_test/ippTools/src/laptool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/laptool.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/laptool.h (revision 33415) @@ -19,4 +19,5 @@ LAPTOOL_MODE_STACKS, LAPTOOL_MODE_UPDATEEXP, + LAPTOOL_MODE_DIFFCHECK, LAPTOOL_MODE_INACTIVEEXP } laptoolMode; Index: branches/meh_branches/ppstack_test/ippTools/src/laptoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/laptoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/laptoolConfig.c (revision 33415) @@ -74,4 +74,5 @@ ADD_OPT(Str, pendingrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,pendingrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, pendingrunArgs, "-limit", "limit result set to N items", 0); // -listrun @@ -86,4 +87,5 @@ ADD_OPT(Str, listrunArgs, "-fault", "search by LAP run fault", NULL); ADD_OPT(Bool,listrunArgs, "-simple", "use the simple output format", false); + ADD_OPT(U64, listrunArgs, "-limit", "limit result set to N items", 0); // -updaterun @@ -132,4 +134,14 @@ ADD_OPT(Bool,updateexpArgs, "-inactive", "set this exposure to active", 0); + // -diffcheck + psMetadata *diffcheckArgs = psMetadataAlloc(); + ADD_OPT(S64, diffcheckArgs, "-lap_id", "search by lap run ID", 0); + ADD_OPT(S64, diffcheckArgs, "-exp_id", "search by exposure ID", 0); + ADD_OPT(S64, diffcheckArgs, "-chip_id", "search by chip ID", 0); + ADD_OPT(S64, diffcheckArgs, "-warp_id", "search by warp ID", 0); + ADD_OPT(S64, diffcheckArgs, "-seq_id", "search by lap sequence ID", 0); + ADD_OPT(Str, diffcheckArgs, "-skycell_id", "search by skycell ID", 0); + ADD_OPT(Bool, diffcheckArgs, "-simple", "use simple output format", 0); + // -inactiveexp psMetadata *inactiveexpArgs = psMetadataAlloc(); @@ -152,4 +164,5 @@ PXOPT_ADD_MODE("-stacks", "", LAPTOOL_MODE_STACKS, stacksArgs); PXOPT_ADD_MODE("-updateexp", "", LAPTOOL_MODE_UPDATEEXP, updateexpArgs); + PXOPT_ADD_MODE("-diffcheck", "", LAPTOOL_MODE_DIFFCHECK, diffcheckArgs); PXOPT_ADD_MODE("-inactiveexp", "", LAPTOOL_MODE_INACTIVEEXP, inactiveexpArgs); Index: branches/meh_branches/ppstack_test/ippTools/src/magicdstool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/magicdstool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/magicdstool.c (revision 33415) @@ -53,4 +53,5 @@ static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state); +static bool validDSRunState(const char *state); # define MODECASE(caseName, func) \ @@ -350,9 +351,9 @@ PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", true, false); // Optional PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); - PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); @@ -374,5 +375,9 @@ pxAddLabelSearchArgs (config, where, "-magic_label", "magicRun.label", "=="); // define magic label psString labelName = NULL; // Name of label - psStringAppend(&labelName, "%sRun.label", stage); + if (strcmp(stage, "camera")) { + psStringAppend(&labelName, "%sRun.label", stage); + } else { + psStringAppend(&labelName, "camRun.label"); + } pxAddLabelSearchArgs (config, where, "-stage_label", labelName, "=="); // define stageRun label psFree(labelName); @@ -392,4 +397,6 @@ break; case IPP_STAGE_CAMERA: + query = pxDataGet("magicdstool_definecopy_camera.sql"); + break; case IPP_STAGE_DIFF: case IPP_STAGE_FAKE: @@ -1206,6 +1213,6 @@ psMetadata *where = psMetadataAlloc(); - // new state (required - PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", true, false); + // new state + PXOPT_LOOKUP_STR(new_state, config->args, "-set_state", false, false); // old state (required) PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); @@ -1231,4 +1238,6 @@ if (!strcmp(state, "failed_revert")) { new_state = "new"; + } else if (!strcmp(state, "failed_revert_ud")) { + new_state = "update"; } else if (!strcmp(state, "failed_cleanup")) { new_state = "goto_cleaned"; @@ -1237,7 +1246,12 @@ return false; } + } else { + if (!validDSRunState(new_state)) { + psError(PS_ERR_UNKNOWN, true, "unexpected value for new state: %s", new_state); + return false; + } } if (!p_psDBRunQueryF(config->dbh, query, new_state)) { - psError(PS_ERR_UNKNOWN, false, "failed to revert"); + psError(PS_ERR_UNKNOWN, false, "failed to clear state faults"); psFree(query); return false; @@ -1397,4 +1411,5 @@ (strcmp(state, "wait") == 0) || (strcmp(state, "failed_revert") == 0) || + (strcmp(state, "failed_revert_ud") == 0) || (strcmp(state, "error_cleaned") == 0) || (strcmp(state, "restored") == 0) || Index: branches/meh_branches/ppstack_test/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/magicdstoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/magicdstoolConfig.c (revision 33415) @@ -79,5 +79,5 @@ psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); psMetadataAddBool(definecopyArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); - psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); @@ -163,5 +163,5 @@ // -clearstatefaults psMetadata *clearstatefaultsArgs = psMetadataAlloc(); - psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL); psMetadataAddStr(clearstatefaultsArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); psMetadataAddS64(clearstatefaultsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0); Index: branches/meh_branches/ppstack_test/ippTools/src/magictool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/magictool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/magictool.c (revision 33415) @@ -131,5 +131,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - + PXOPT_LOOKUP_BOOL(multidiff, config->args, "-multidiff", false); psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "=="); @@ -142,5 +142,11 @@ // Get list of exposures ready to magic { - psString query = pxDataGet("magictool_definebyquery_select.sql"); + psString query; + if (multidiff) { + query = pxDataGet("magictool_definebyquery_select_multidiff.sql"); + } + else { + query = pxDataGet("magictool_definebyquery_select.sql"); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); Index: branches/meh_branches/ppstack_test/ippTools/src/magictoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/magictoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/magictoolConfig.c (revision 33415) @@ -61,4 +61,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-multidiff", 0, "allow multiple diffs to be magicked with the same label", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false); Index: branches/meh_branches/ppstack_test/ippTools/src/mergetool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/mergetool.c (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/mergetool.c (revision 33415) @@ -0,0 +1,1227 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingmergeMode(pxConfig *config); +static bool addmergedMode(pxConfig *config); +static bool listmergedMode(pxConfig *config); +static bool revertmergedMode(pxConfig *config); +static bool updatemergedMode(pxConfig *config); +static bool definebyquerymergecopyMode(pxConfig *config); +static bool listmergedvodbcopyMode(pxConfig *config); +static bool revertmergedvodbcopyMode(pxConfig *config); +static bool updatemergedvodbcopyMode(pxConfig *config); + + + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = mergetoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MERGETOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(MERGETOOL_MODE_PENDINGMERGE, pendingmergeMode); + MODECASE(MERGETOOL_MODE_ADDMERGED, addmergedMode); + MODECASE(MERGETOOL_MODE_LISTMERGED, listmergedMode); + MODECASE(MERGETOOL_MODE_REVERTMERGED, revertmergedMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGED, updatemergedMode); + MODECASE(MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyMode); + MODECASE(MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyMode); + MODECASE(MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyMode); + + + + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); + PXOPT_COPY_STR(config->args, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); + // PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + //find the items to queue (uniq minidvodb_id) + psString dvodb_string = NULL; + psString bare_query = NULL; + +bare_query = pxDataGet("mergetool_find_minidvodb_id.sql"); + psStringAppend(&dvodb_string, "mergedvodbRun.mergedvodb = '%s'", mergedvodb); + +if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + psFree(where); + + + psStringAppend(&query, " GROUP BY minidvodb_id"); + + + psTrace("mergetool.c", PS_LOG_INFO,"query: \n\n%s\n\n",query); + +if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error, \n%s\n", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + if (!mergedvodb_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue merge run without a defined mergedvodb_path: minidvodb_id %" PRId64,row->minidvodb_id); + psFree(output); + return false; + } + + psFree(row); + +} + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } +for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 minidvodb_id =0; + + minidvodbRunRow *row = minidvodbRunObjectFromMetadata(md); + minidvodb_id = row->minidvodb_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into minidvodbRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxmergeQueueByMinidvodbID(config, + minidvodb_id, + mergedvodb, + mergedvodb_path + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue mergedvodb %" PRId64, minidvodb_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + + } + +psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(mergedvodb_path, config->args, "-set_mergedvodb_path", false, false); + psString query = NULL; + + query = psStringCopy("UPDATE mergedvodbRun SET"); + + int cnt = 0; + psString comma = ","; + if (state) { + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + if (mergedvodb_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodb_path = '%s'", mergedvodb_path); + cnt++; + } + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + + return true; +} + +static bool pendingmergeMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_pending_merge.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbPendingMerge", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool addmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", true, false); + PXOPT_LOOKUP_STR(mergedvodb, config->args, "-mergedvodb", true, false); + + // optional + +//PXOPT_LOOKUP_U64(merge_order, config->args, "-merge_order", false, false); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + + PXOPT_LOOKUP_TIME(epoch, config->args, "-epoch", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psString query = pxDataGet("mergetool_find_pendingmergeprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + psTrace("mergetool.c", PS_LOG_INFO, "query is:\n%s", query); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + +psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + mergedvodbRunRow *pendingRow = mergedvodbRunObjectFromMetadata(output->data[0]); + psFree(output); + mergedvodbProcessedRow *row = mergedvodbProcessedRowAlloc( + pendingRow->merge_id, + 0, + dtime_verify ? dtime_verify : 0, + dtime_merge ? dtime_merge : 0, + dtime_script ? dtime_script : 0, + epoch, + fault + ); + + if (!mergedvodbProcessedInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + +//this finds the # of merged things (for the merge order) + psString query3 = NULL; + psStringAppend(&query3, "select count(*) from mergedvodbRun join mergedvodbProcessed using (merge_id) where state = 'full' and mergedvodb = '%s';", mergedvodb); + + if (!p_psDBRunQuery(config->dbh, query3)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query3); + return false; + } + psArray *output2 = p_psDBFetchResult(config->dbh); + if (!output2) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output2)) { + psTrace("mergetool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + bool status; + psS64 m_order = psMetadataLookupS64(&status, output2->data[0], "count(*)"); + if (!status) { + + psAbort("failed to lookup value for count column"); + return false; + } + psString final = NULL; + psStringAppend(&final, "%" PRIu64, m_order); + //return false; + psFree(query3); + psFree(output2); + + + + //update the merge_order + + psString query4 = NULL; + psStringAppend(&query4, "update mergedvodbProcessed set merge_order = %"PRIu64,m_order); + psStringAppend(&query4," where merge_id = %" PRIu64, merge_id); + //printf("%s", query4); + if (!p_psDBRunQuery(config->dbh, query4)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query4); + return false; + } + // +psFree(query4); + +// since there is only one exp per 'new' set mergedvodbRun.state = 'full' + + psString query2 = NULL ; + + if (!fault) { + psStringAppend(&query2, "UPDATE mergedvodbRun SET state = 'full' WHERE merge_id = %'" PRIu64, row->merge_id); + + if (!p_psDBRunQuery(config->dbh, query2)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query2); + return false; + } + + } + + + + + + + psFree(row); + psFree(pendingRow); + + + + + + //commit the changes + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + + //print the merge_order (why not!) + printf("%s", final); + psFree(final); + + + + + + + + + + + + return true; +} + + +static bool listmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "mergedvodbRun.minidvodb_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + psString query = NULL; + query = pxDataGet("mergetool_find_processed.sql"); + +if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + + + // we either add AND (condition) or WHERE (condition): + if (psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND fault != 0"); + } + if (psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND fault = 0"); + } + if (!psListLength(where->list) && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE fault != 0"); + } + if (!psListLength(where->list) && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE fault = 0"); + } + +psFree(where); + + // order by merge_id + psStringAppend(&query, " ORDER BY merge_id"); + + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + + + return true; +} + + +static bool revertmergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbRun.fault", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = NULL; + query = pxDataGet("mergetool_revertmergedvodbprocessed.sql"); + +if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + + + return true; +} + +static bool updatemergedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(minidvodb_id, config->args, "-merge_id", true, false); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbProcessed.merge_id", "=="); + PXOPT_LOOKUP_F32(dtime_merge, config->args, "-set_dtime_merge", false, false); + PXOPT_LOOKUP_F32(dtime_verify, config->args, "-set_dtime_verify", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psString query = psStringCopy("UPDATE mergedvodbProcessed JOIN mergedvodbRun USING (merge_id) SET "); +int cnt = 0; + psString comma = ","; + if (fault) { + psStringAppend(&query, " fault = %d", fault); + cnt++; + } + + + + if (dtime_verify) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_verify = %f", dtime_verify); + cnt++; + } + + if (dtime_merge) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_merge = %f", dtime_merge); + cnt++; + } + +if (dtime_script) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " dtime_script = %f", dtime_script); + cnt++; + } +if (state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " state = '%s'", state); + cnt++; + } + + + + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error \n%s\n",query ); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + +static bool definebyquerymergecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbRun.merge_id", "=="); + pxAddLabelSearchArgs (config, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); // define using camRun label + + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(destination_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(mergedvodb_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(last_merged, config->args, "-last_merged", false); + // prevent queueing an addRun if a given exposure has already been added to + // the given dvo database + psString dvodb_string = NULL; + psString bare_query = NULL; + if (destination_host) { + psTrace("mergedvodbtool.c", PS_LOG_INFO, "destination_host argument found (%s) using mergedvodbtool_find_merge_id_dvo.sql\n", destination_host); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("mergedvodbtool_find_merge_id_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "mergedvodbCopy.destination_host = '%s'", destination_host); + } else { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host"); + + return false; + } + + if (!bare_query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); + psFree(where); + return false; + } + // Take the bare query and add the dvodb selector + psString query = NULL; + psStringAppend(&query, bare_query, dvodb_string); + psFree(dvodb_string); + psFree(bare_query); + + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + if (last_merged) { + psStringAppend(&query, " ORDER BY merge_order DESC LIMIT 1"); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into mergedvodbRun"); + psFree(output); + return false; + } + + if (!mergedvodb_rsync_path) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined mergedvodb_rsync_path: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + if (!destination_host) { + psError(PS_ERR_UNKNOWN, false, "cannot queue mergedvodbcopy run without a defined destination_host: merge_id %" PRId64, row->merge_id); + psFree(output); + return false; + } + + psFree(row); + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + mergedvodbRunRow *row = mergedvodbRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByMergeID(config, + row->merge_id, + destination_host ? destination_host : "NULL", + mergedvodb_rsync_path ?mergedvodb_rsync_path : "NULL" + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue merge_id: %" PRId64, row->merge_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool listmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb", "mergedvodbRun.mergedvodb", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pending, config->args, "-pending", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("mergedvodbtool_find_mergedvodbcopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + +if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + +// we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + + // list only faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND mergedvodbCopy.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE mergedvodbCopy.fault = 0"); + } + psFree(where); + + if (pending) { + //add the cuts for pending (state new, no faults) + psStringAppend(&query, " %s", " AND mergedvodbCopy.state = 'new' AND mergedvodbCopy.fault = 0 AND mergedvodbCopy.destination_host IS NOT NULL AND mergedvodbCopy.mergedvodb_rsync_path IS NOT NULL"); + } + + + // order by epoch + psStringAppend(&query, " ORDER BY mergedvodbcopy_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s ", query); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "mergedvodbCopy", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + +return true; +} + +static bool revertmergedvodbcopyMode(pxConfig *config) { + psMetadata *where = psMetadataAlloc(); + PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + PXOPT_COPY_STR(config->args, where, "-destination_host", "destination_host", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "mergedvodbCopy.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("mergedvodbtool_revertmergedvodbcopy.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + } + psError(PS_ERR_UNKNOWN, false, "database error %s", query ); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool updatemergedvodbcopyMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_LOOKUP_U64(mergedvodbcopy_id, config->args, "-mergedvodbcopy_id", false, false); + PXOPT_LOOKUP_U64(merge_id, config->args, "-merge_id", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + PXOPT_LOOKUP_STR(host, config->args, "-host", false, false); + PXOPT_LOOKUP_STR(rsync_path, config->args, "-mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + PXOPT_LOOKUP_S16(set_fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_STR(set_rsync_path, config->args, "-set_mergedvodb_rsync_path", false, false); + PXOPT_LOOKUP_STR(set_host, config->args, "-set_destination_host", false, false); + PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_F32(dtime, config->args, "-set_dtime", false, false); + PXOPT_COPY_S64(config->args, where, "-mergedvodbcopy_id", "mergedvodbCopy.mergedvodbcopy_id", "=="); + PXOPT_COPY_S64(config->args, where, "-merge_id", "mergedvodbCopy.merge_id", "=="); + + PXOPT_COPY_STR(config->args, where, "-state", "mergedvodbCopy.state", "=="); + PXOPT_COPY_STR(config->args, where, "-host", "mergedvodbCopy.destination_host", "=="); + PXOPT_COPY_STR(config->args, where, "-mergedvodb_rsync_path", "mergedvodbCopy.mergedvodb_rsync_path", "=="); + + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE mergedvodbCopy JOIN mergedvodbRun USING (merge_id) SET "); + int cnt = 0; + psString comma = ","; + if (set_fault) { + psStringAppend(&query, " mergedvodbCopy.fault = %d", set_fault); + cnt++; + } + if (set_rsync_path) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.mergedvodb_rsync_path = '%s'" , set_rsync_path); + cnt++; + } + + if (set_host) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.destination_host = '%s'" , set_host); + cnt++; + } + + + if (set_state) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + + psStringAppend(&query, " mergedvodbCopy.state = '%s'" , set_state); + cnt++; + } + + if (dtime) { + if (cnt) { + psStringAppend(&query, "%s", comma); + } + psStringAppend(&query, " mergedvodbCopy.dtime = %f", dtime); + cnt++; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + psFree(query); + return false; + } + + psFree(query); + psFree(where); + + return true; +} + + + Index: branches/meh_branches/ppstack_test/ippTools/src/mergetool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/mergetool.h (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/mergetool.h (revision 33415) @@ -0,0 +1,41 @@ +/* + * mergetool.h + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef MERGETOOL_H +#define MERGETOOL_H 1 + +#include "pxtools.h" + +typedef enum { + MERGETOOL_MODE_NONE = 0x0, + MERGETOOL_MODE_DEFINEBYQUERY, + MERGETOOL_MODE_UPDATERUN, + MERGETOOL_MODE_PENDINGMERGE, + MERGETOOL_MODE_ADDMERGED, + MERGETOOL_MODE_LISTMERGED, + MERGETOOL_MODE_REVERTMERGED, + MERGETOOL_MODE_UPDATEMERGED, + MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, + MERGETOOL_MODE_LISTMERGEDVODBCOPY, + MERGETOOL_MODE_REVERTMERGEDVODBCOPY, + MERGETOOL_MODE_UPDATEMERGEDVODBCOPY +} mergetoolMode; + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv); + +#endif // MERGETOOL_H Index: branches/meh_branches/ppstack_test/ippTools/src/mergetoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/mergetoolConfig.c (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/mergetoolConfig.c (revision 33415) @@ -0,0 +1,196 @@ +/* + * addtoolConfig.c + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxmerge.h" +#include "mergetool.h" + +pxConfig *mergetoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + // pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodbRun.mergedvodb and use as mergedvodb", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-minidvodb_group", PS_META_DUPLICATE_OK, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + // psMetadataAddStr(definebyqueryArge, PS_LIST_TAIL, "-set_mergedvodb", 0, "define mergedvodb"); + + // psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadata *updaterunArgs = psMetadataAlloc(); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", 0); + psMetadataAddU64(updaterunArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_mergedvodb_path", 0, "define workdir", NULL); + + + psMetadata *pendingmergeArgs = psMetadataAlloc(); + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(pendingmergeArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadataAddU64(pendingmergeArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(pendingmergeArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + + + psMetadata *addmergedArgs = psMetadataAlloc(); + psMetadataAddU64(addmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id (required)", 0); + psMetadataAddStr(addmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb (required)", 0); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_verify", 0, "define elapsed time for DVO verify (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_merge", 0, "define elapsed time for DVO merge (seconds)", NAN); + psMetadataAddF32(addmergedArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time for script (seconds)", NAN); + psMetadataAddTime(addmergedArgs, PS_LIST_TAIL, "-epoch", 0, "time merge is finished", NULL); + psMetadataAddS16(addmergedArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + + + + + psMetadata *listmergedArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-minidvodb_id", 0, "search by minidvodb_id", 0); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", NULL); + psMetadataAddStr(listmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by minidvodbRun.minidvodb_group", NULL); + psMetadataAddU64(listmergedArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + + psMetadata *revertmergedArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(revertmergedArgs, PS_LIST_TAIL, "-fault", 0, "search by minidvodbCopyfault", 0); + psMetadataAddStr(revertmergedArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodb", NULL); + + psMetadata *updatemergedArgs = psMetadataAlloc(); + + psMetadataAddU64(updatemergedArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddS16(updatemergedArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_verify", 0, "define elapsed time for DVO verify (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_merge", 0, "define elapsed time for DVO merge (seconds)", 0); + psMetadataAddF32(updatemergedArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updatemergedArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + // -definebyquerymergecopy + psMetadata *definebyquerymergecopyArgs = psMetadataAlloc(); + psMetadataAddS64(definebyquerymergecopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-mergedvodb", PS_META_DUPLICATE_OK, "search by mergedvodb", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "define workdir", NULL); + psMetadataAddStr(definebyquerymergecopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "set destination host", NULL); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(definebyquerymergecopyArgs, PS_LIST_TAIL, "-last_merged", 0, "if multiple minidvodbs have been dvomerged between rsyncs, then only queue the most recent merged dvodb for copying", false); + + psMetadata *listmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by mergedvodb_id", 0); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb", 0, "search by mergedvodbRun.mergedvodb", NULL); + psMetadataAddStr(listmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by mergedvodbCopy.destination_host", NULL); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-pending", 0, "limit to pending items", false); + psMetadataAddU64(listmergedvodbcopyArgs, PS_LIST_TAIL, "-limit", 0, "limit to N items", 0); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-simple", 0, "simple output", false); + psMetadataAddBool(listmergedvodbcopyArgs, PS_LIST_TAIL, "-faulted", 0, "limit to faulted state", false); + + psMetadata *revertmergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-destination_host", 0, "search by destination_host", NULL); + psMetadataAddS16(revertmergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by mergedvodbCopyfault", 0); + psMetadata *updatemergedvodbcopyArgs = psMetadataAlloc(); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodbcopy_id", 0, "search by mergedvodbcopy_id", 0); + psMetadataAddU64(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-merge_id", 0, "search by merge_id", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-host", 0, "search by host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-mergedvodb_rsync_path", 0, "search by rsync", NULL); + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + + psMetadataAddS16(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + psMetadataAddF32(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_dtime", 0, "set elapsed time for transfer", 0); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_mergedvodb_rsync_path", 0, "change the mergedvodb_rsync_path", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_destination_host", 0, "change the host", NULL); + psMetadataAddStr(updatemergedvodbcopyArgs, PS_LIST_TAIL, "-set_state", 0, "change the state", NULL); + + + + + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from minindvodbRun", MERGETOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "update existing mergedvodbRuns", MERGETOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingmerge", "list pending mergedvodbRuns", MERGETOOL_MODE_PENDINGMERGE, pendingmergeArgs); + PXOPT_ADD_MODE("-addmerged", "add processed mergedvodbRun", MERGETOOL_MODE_ADDMERGED, addmergedArgs); + PXOPT_ADD_MODE("-listmerged","list merged dvodbs", MERGETOOL_MODE_LISTMERGED, listmergedArgs); + PXOPT_ADD_MODE("-revertmerged","revert merged dvodbs", MERGETOOL_MODE_REVERTMERGED, revertmergedArgs); + PXOPT_ADD_MODE("-updatemerged","change merged dvodb properties",MERGETOOL_MODE_UPDATEMERGED, updatemergedArgs); + PXOPT_ADD_MODE("-definebyquerymergecopy", "create runs from mergedvodbRun", MERGETOOL_MODE_DEFINEBYQUERYMERGECOPY, definebyquerymergecopyArgs); + PXOPT_ADD_MODE("-listmergedvodbcopy","list copy mergedvodbs", MERGETOOL_MODE_LISTMERGEDVODBCOPY, listmergedvodbcopyArgs); + PXOPT_ADD_MODE("-revertmergedvodbcopy","revert copy mergedvobs",MERGETOOL_MODE_REVERTMERGEDVODBCOPY, revertmergedvodbcopyArgs); + PXOPT_ADD_MODE("-updatemergedvodbcopy","change mergedvodb copy properties",MERGETOOL_MODE_UPDATEMERGEDVODBCOPY, updatemergedvodbcopyArgs); + + + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/meh_branches/ppstack_test/ippTools/src/minidvodbtool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/minidvodbtool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/minidvodbtool.c (revision 33415) @@ -89,5 +89,5 @@ psMetadata *where = psMetadataAlloc(); pxcamGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodbRun_id", "=="); + PXOPT_COPY_S64(config->args, where, "-minidvodb_id", "minidvodbRun.minidvodb_id", "=="); pxAddLabelSearchArgs (config, where, "-minidvodb_group", "minidvodbRun.minidvodb_group", "=="); // define using camRun label Index: branches/meh_branches/ppstack_test/ippTools/src/pstamptool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pstamptool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pstamptool.c (revision 33415) @@ -56,4 +56,7 @@ static bool revertdependentMode(pxConfig *config); static bool getwebrequestnumMode(pxConfig *config); +static bool addfileMode(pxConfig *config); +static bool listfileMode(pxConfig *config); +static bool deletefileMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -103,4 +106,7 @@ MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode); MODECASE(PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumMode); + MODECASE(PSTAMPTOOL_MODE_ADDFILE, addfileMode); + MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode); + MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode); default: psAbort("invalid option (this should not happen)"); @@ -173,6 +179,6 @@ if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -698,6 +704,8 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); - PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); + PXOPT_LOOKUP_S64(parent_id, config->args, "-parent_id", false, false); + PXOPT_LOOKUP_BOOL(is_parent, config->args, "-is_parent", false); // unless the job is being inserted with stop state require outputBase @@ -710,4 +718,19 @@ if (!strcmp(job_type, "get_image") || !strcmp(job_type, "detect_query") || !strcmp(job_type, "none")) { stampJob = false; + } else if (!strcmp(job_type, "child")) { + // job_type child's only action is to resolve a dependent + // XXX: IS this necessary? + if (!dep_id) { + psError(PS_ERR_UNKNOWN, true, "dep_id required for child job\n"); + return false; + } + if (is_parent) { + psError(PS_ERR_UNKNOWN, true, "job type child can not be a parent job\n"); + return false; + } + stampJob = false; + } else if (!strcmp(job_type, "mosaic")) { + stampJob = false; + is_parent = true; } else if (!strcmp(job_type, "stamp")) { stampJob = true; @@ -717,5 +740,5 @@ } if (!pstampJobInsert(config->dbh, - 0, // job_id + 0, // job_id req_id, rownum, @@ -727,5 +750,7 @@ options, dep_id, - 0 // fault_count + 0, // fault_count + parent_id, + is_parent )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -755,4 +780,5 @@ PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "=="); + PXOPT_COPY_STR(config->args, where, "-jobType", "jobType", "=="); PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); @@ -836,8 +862,11 @@ // use psDBGenerateWhereSQL because the SQL yields an intermediate table + psString whereStr = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereStr, "\n AND %s", whereClause); psFree(whereClause); + } else { + psStringAppend(&whereStr, "%s", ""); } psFree(where); @@ -852,9 +881,11 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereStr); + psFree(query); + return false; + } + psFree(whereStr); psFree(query); @@ -950,4 +981,5 @@ PXOPT_LOOKUP_S32(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); psMetadata *where = psMetadataAlloc(); @@ -980,5 +1012,5 @@ psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, fault, fault)) { + if (!p_psDBRunQueryF(config->dbh, query, fault, fault, state)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -1476,2 +1508,122 @@ return true; } + +static bool addfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_STR(path, config->args, "-path", true, false); + + if (!pstampFileInsert(config->dbh, + 0, // file_id + job_id, + path + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 file_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", file_id); + + return true; +} + +static bool listfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-file_id", "file_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psListLength(where->list)) { + fprintf(stderr, "search arguments are required\n"); + exit (1); + } + + psString query = pxDataGet("pstamptool_listfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampFile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool deletefileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + + psString query = NULL; + psStringAppend(&query, "DELETE FROM pstampFile WHERE job_id = %" PRId64, job_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("pstamptool", PS_LOG_INFO, "Deleted %" PRIu64 " rows from pstampFile", affected); + + return true; +} Index: branches/meh_branches/ppstack_test/ippTools/src/pstamptool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pstamptool.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pstamptool.h (revision 33415) @@ -50,4 +50,7 @@ PSTAMPTOOL_MODE_REVERTDEPENDENT, PSTAMPTOOL_MODE_GETWEBREQUESTNUM, + PSTAMPTOOL_MODE_ADDFILE, + PSTAMPTOOL_MODE_LISTFILE, + PSTAMPTOOL_MODE_DELETEFILE, } pstamptoolMode; Index: branches/meh_branches/ppstack_test/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pstamptoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pstamptoolConfig.c (revision 33415) @@ -144,4 +144,6 @@ psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-parent_id", 0, "define parent's job_id", 0); + psMetadataAddBool(addjobArgs, PS_LIST_TAIL, "-is_parent", 0, "define whether job has children", false); psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); @@ -152,4 +154,5 @@ psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-dep_id", 0, "select by dependent ID", 0); psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddStr(listjobArgs, PS_LIST_TAIL, "-jobType", 0, "select by jobType", 0); psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -186,4 +189,5 @@ psMetadataAddS32(stopdependentjobArgs, PS_LIST_TAIL, "-fault_count", 0, "select by fault_count (>=)", 0); psMetadataAddS16(stopdependentjobArgs, PS_LIST_TAIL, "-set_fault", 0, "new fault value for job and dependent (required)", 0); + psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-set_state", 0, "new pstampDependent.state", "new"); psMetadataAddStr(stopdependentjobArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL); psMetadataAddU64(stopdependentjobArgs, PS_LIST_TAIL, "-limit", 0, "not used", 0); @@ -274,4 +278,21 @@ psMetadata *modes = psMetadataAlloc(); + // -addfile + psMetadata *addfileArgs = psMetadataAlloc(); + psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-job_id", 0, "define job ID for file (required)", 0); + psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-path", 0, "define path for file (required)", NULL); + + // -listfile + psMetadata *listfileArgs = psMetadataAlloc(); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-file_id", 0, "select by file ID", 0); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddU64(listfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -deletefile + psMetadata *deletefileArgs = psMetadataAlloc(); + psMetadataAddS64(deletefileArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID (required)", 0); + PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); @@ -302,4 +323,7 @@ PXOPT_ADD_MODE("-project", "", PSTAMPTOOL_MODE_PROJECT, projectArgs); PXOPT_ADD_MODE("-getwebrequestnum","", PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumArgs); + PXOPT_ADD_MODE("-addfile", "", PSTAMPTOOL_MODE_ADDFILE, addfileArgs); + PXOPT_ADD_MODE("-listfile", "", PSTAMPTOOL_MODE_LISTFILE, listfileArgs); + PXOPT_ADD_MODE("-deletefile", "", PSTAMPTOOL_MODE_DELETEFILE, deletefileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/meh_branches/ppstack_test/ippTools/src/pxadd.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pxadd.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pxadd.c (revision 33415) @@ -131,4 +131,5 @@ char *stage, psS64 stage_id, + psS32 stage_extra1, char *workdir, char *reduction, @@ -159,4 +160,8 @@ psMemSetPersistent(query, true); } + if (strcmp(stage,"staticsky_multi") == 0) { + query = pxDataGet("addtool_queue_sky_id_multi.sql"); + psMemSetPersistent(query, true); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -170,4 +175,5 @@ // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, + stage_extra1 ? stage_extra1 : 0, "new", // state workdir ? workdir : "NULL", Index: branches/meh_branches/ppstack_test/ippTools/src/pxadd.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pxadd.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pxadd.h (revision 33415) @@ -33,4 +33,5 @@ char *stage, psS64 stage_id, + psS32 stage_extra1, char *workdir, char *reduction, Index: branches/meh_branches/ppstack_test/ippTools/src/pxchip.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pxchip.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pxchip.c (revision 33415) @@ -71,5 +71,5 @@ psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN); psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN); - psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); + psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); psMetadataAddStr(md, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL); Index: branches/meh_branches/ppstack_test/ippTools/src/pxmerge.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pxmerge.c (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/pxmerge.c (revision 33415) @@ -0,0 +1,97 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmerge.h" + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodb, + char *mergedvodb_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_merge_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbRun SELECT 0, minidvodb_id, '%s', '%s', 'new', NULL FROM minidvodbRun WHERE minidvodbRun.minidvodb_id = %lld and minidvodbRun.state = 'merged'", mergedvodb, mergedvodb_path, (long long) minidvodb_id)) + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + return true; +} + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("mergetool_queue_mergecopy_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + } + + if (!p_psDBRunQueryF(config->dbh,"INSERT INTO mergedvodbCopy SELECT 0, merge_id, '%s', '%s', 0, 'new', NULL, 0 FROM mergedvodbRun WHERE mergedvodbRun.merge_id = %lld and (mergedvodbRun.state = 'merged' or mergedvodbRun.state = 'full')", mergedvodb_rsync_path, destination_host, (long long) merge_id)) + + { + psError(PS_ERR_UNKNOWN, false, "database error %s", query); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row \n%s", query); + return false; + } + + return true; +} Index: branches/meh_branches/ppstack_test/ippTools/src/pxmerge.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pxmerge.h (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/pxmerge.h (revision 33415) @@ -0,0 +1,41 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMERGE_H +#define PXMERGE_H 1 + +#include + +#include "pxtools.h" + + +bool pxmergeQueueByMinidvodbID(pxConfig *config, + psS64 minidvodb_id, + char *mergedvodbb, + char *mergedvodb_path + ); + +bool pxaddQueueByMergeID(pxConfig *config, + psS64 merge_id, + char *destination_host, + char *mergedvodb_rsync_path + ); + + +#endif // PXMERGE_H Index: branches/meh_branches/ppstack_test/ippTools/src/pxtools.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/pxtools.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/pxtools.c (revision 33415) @@ -35,4 +35,5 @@ if (!strcmp(state, "drop")) return true; if (!strcmp(state, "wait")) return true; + if (!strcmp(state, "keep")) return true; if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; Index: branches/meh_branches/ppstack_test/ippTools/src/regtool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/regtool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/regtool.c (revision 33415) @@ -268,4 +268,8 @@ } } + char *is_stareim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'a'); + if (is_stareim) { + continue; + } if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)|| @@ -386,4 +390,9 @@ } } + char *is_stareim = strchr(psMetadataLookupStr(NULL,row,"exp_name"),'a'); + if (is_stareim) { + continue; + } + bool status = false; @@ -882,12 +891,18 @@ PXOPT_LOOKUP_STR(set_md5sum, config->args, "-set_md5sum", false, false); PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_BOOL(set_ignored, config->args, "-set_ignored", false); + PXOPT_LOOKUP_BOOL(clear_ignored, config->args, "-clear_ignored", false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected"); + if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state && !set_ignored && !clear_ignored) { + psError(PS_ERR_UNKNOWN, false, "at least one of -fault or -burntool_state -set_ignored -clear_ignored or -set_state must be selected"); return false; } if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) { psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); + return false; + } + if (set_ignored && clear_ignored) { + psError(PS_ERR_UNKNOWN, true, "only one of -set_ignored or -clear_ignored may be selected"); return false; } @@ -898,5 +913,6 @@ if (fault != INT16_MAX) { - // this is fairly dangerous : can set all if the where is not set... + // this is fairly dangerous : can set all if the where is not set... but since exp_id and class_id + // were required above this is safe if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault, 0)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); @@ -930,4 +946,16 @@ } psStringAppend(&setvalues,"rawImfile.data_state = '%s'",set_state); + } + if (set_ignored) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.ignored = 1"); + } + if (clear_ignored) { + if (setvalues) { + psStringAppend(&setvalues,","); + } + psStringAppend(&setvalues,"rawImfile.ignored = 0"); } if (hostname) { Index: branches/meh_branches/ppstack_test/ippTools/src/regtoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/regtoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/regtoolConfig.c (revision 33415) @@ -194,4 +194,6 @@ ADD_OPT(Str, updateprocessedimfileArgs, "-set_md5sum", "set md5sum", NULL); ADD_OPT(Str, updateprocessedimfileArgs, "-set_state", "set data state", NULL); + psMetadataAddBool(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_ignored", 0, "set imfile to be ignored for processing", false); + psMetadataAddBool(updateprocessedimfileArgs, PS_LIST_TAIL, "-clear_ignored", 0, "set imfile to not be ignored for processing", false); Index: branches/meh_branches/ppstack_test/ippTools/src/staticskytool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/staticskytool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/staticskytool.c (revision 33415) @@ -38,7 +38,15 @@ static bool resultMode(pxConfig *config); static bool revertMode(pxConfig *config); -static bool updateresult(pxConfig *config); +static bool updateresultMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); + +static bool defineskycalrunMode(pxConfig *config); +static bool updateskycalrunMode(pxConfig *config); +static bool pendingskycalrunMode(pxConfig *config); +static bool addskycalresultMode(pxConfig *config); +static bool skycalresultMode(pxConfig *config); +static bool revertskycalresultMode(pxConfig *config); +static bool updateskycalresultMode(pxConfig *config); static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state); @@ -69,7 +77,14 @@ MODECASE(STATICSKYTOOL_MODE_RESULT, resultMode); MODECASE(STATICSKYTOOL_MODE_REVERT, revertMode); - MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresult); + MODECASE(STATICSKYTOOL_MODE_UPDATERESULT, updateresultMode); MODECASE(STATICSKYTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(STATICSKYTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunMode); + MODECASE(STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_UPDATESKYCALRESULT,updateskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode); + MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultMode); default: psAbort("invalid option (this should not happen)"); @@ -97,10 +112,8 @@ { PS_ASSERT_PTR_NON_NULL(config, false); - - // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); // optional - PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); @@ -111,5 +124,6 @@ psMetadata *whereMD = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); @@ -621,5 +635,8 @@ PXOPT_COPY_STR(config->args, where, "-label", "staticskyRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "staticskyRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + PXOPT_LOOKUP_S32(num_filters, config->args, "-num_filters", false, false); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -645,8 +662,14 @@ psFree(where); + psStringAppend(&query, "\nGROUP BY sky_id"); + if (num_filters) { + psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters); + } + + // treat limit == 0 as "no limit" if (limit) { psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); + psStringAppend(&query, "\n%s", limitString); psFree(limitString); } @@ -735,5 +758,5 @@ } -static bool updateresult(pxConfig *config) +static bool updateresultMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -978,2 +1001,477 @@ # endif +static bool defineskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required options + // none required. We get workdir, etc from staticskyRun if not provided + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + + psMetadata *whereMD = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, whereMD, "-select_sky_id", "staticskyRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-select_stack_id", "stackRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); + PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">="); + pxAddLabelSearchArgs(config, whereMD, "-select_label", "stackRun.label", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psString query = pxDataGet("staticskytool_defineskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(whereMD); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\nAND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + if (!rerun) { + if (label) { + psStringAppend(&query, "\nAND (skycalRun.label IS NULL OR skycalRun.label = '%s')", label); + } + psStringAppend(&query, "\nAND skycal_id IS NULL"); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs + // we now need to loop over all of these and for each pair, select the best set of + // inputs + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(output)) { + psWarning("staticskytool: no rows found"); + psFree(output); + return true; + } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "skycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < output->n; i++) { + psMetadata *row = output->data[i]; // Row from select + bool status; + + psS64 sky_id = psMetadataLookupS64(&status, row, "sky_id"); + psS64 stack_id = psMetadataLookupS64(&status, row, "stack_id"); + psString sky_workdir = psMetadataLookupStr(&status, row, "workdir"); + psString sky_label = psMetadataLookupStr(&status, row, "label"); + psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + + // create a staticskyRun + if (!skycalRunInsert(config->dbh, + 0x0, // skycal_id + sky_id, + stack_id, + "new", // state + workdir ? workdir : sky_workdir, + label ? label : sky_label, + data_group ? data_group : sky_data_group, + dist_group, + reduction, + registered, + note + ) + ) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + } + psFree(output); + return true; +} + +static bool updateskycalrunMode(pxConfig *config) +{ +#ifdef notyet + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE staticskyRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true, false); + psFree(query); + psFree(where); + + return result; +#endif + return false; +} +static bool pendingskycalrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *whereMD = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, whereMD, "-skycal_id", "skycal_id", "=="); + PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "=="); + pxAddLabelSearchArgs (config, whereMD, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, whereMD, "-filter", "stackRun.filter", "LIKE"); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_pendingskycalrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(whereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + psFree(whereMD); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psStringAppend(&query, "\n AND %s", whereClause); + psFree(whereClause); + psFree(whereMD); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, "\n%s", limitString); + psFree(limitString); + } + + // the where clause is required and matches the WHERE hook format string + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingskycalRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + return true; +} + +static bool addskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(skycal_id, config->args, "-skycal_id", true, false); + + // optional + PXOPT_LOOKUP_F32(sigma_ra, config->args, "-sigma_ra", false, false); + PXOPT_LOOKUP_F32(sigma_dec, config->args, "-sigma_dec", false, false); + + PXOPT_LOOKUP_F32(zpt_obs, config->args, "-zpt_obs", false, false); + PXOPT_LOOKUP_F32(zpt_stdev, config->args, "-zpt_stdev", false, false); + + + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_F32(dtime_astrom, config->args, "-dtime_astrom", false, false); + + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); +// PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + PXOPT_LOOKUP_S32(n_astrom, config->args, "-n_astrom", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); + PXOPT_LOOKUP_STR(ver_psastro, config->args, "-ver_psastro", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + + psString software_ver = NULL; + if ((ver_pslib)&&(ver_psmodules)) { + software_ver = pxMergeCodeVersions(ver_pslib,ver_psmodules); + } + if (ver_psphot) { + software_ver = pxMergeCodeVersions(software_ver,ver_psphot); + } + if (ver_psastro) { + software_ver = pxMergeCodeVersions(software_ver,ver_psastro); + } + if (ver_ppstats) { + software_ver = pxMergeCodeVersions(software_ver,ver_ppstats); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + skycalResultRow *row = skycalResultRowAlloc( + skycal_id, + path_base, + dtime_script, + dtime_astrom, + sigma_ra, + sigma_dec, + n_astrom, + zpt_obs, + zpt_stdev, + quality, + software_ver, + hostname, + fault + ); + + if (!skycalResultInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + return false; + } + + psFree(row); + + if (fault) { + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; + } + + psString query = "UPDATE skycalRun SET state = 'full' WHERE skycal_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, skycal_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool updateskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "=="); + + if (!pxSetFaultCode(config->dbh, "skycalResult", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + return true; +} +static bool skycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); + PXOPT_COPY_S64(config->args, where, "-sky_id", "skycalRun.sky_id", "=="); + PXOPT_COPY_S64(config->args, where, "-stack_id", "skycalRun.stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S16(config->args, where, "-fault", "skycalResult.fault", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("staticskytool_skycalresult.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("staticskytool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} +static bool revertskycalresultMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-skycal_id", "staticskyResult.sky_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "staticskyRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-data_group", "staticskyRun.data_group", "=="); + pxAddLabelSearchArgs(config, where, "-filter", "stackRun.filter", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "staticskyResult.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // Delete product + psString delete = pxDataGet("staticskytool_revertskycal.sql"); + if (!delete) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&delete, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, delete)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(delete); + psFree(where); + return false; + } + psFree(delete); + + int numRows = psDBAffectedRows(config->dbh); // Number of row affected + psLogMsg("staticskytool", PS_LOG_INFO, "Deleted %d rows", numRows); + + psFree(where); + return true; +} Index: branches/meh_branches/ppstack_test/ippTools/src/staticskytool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/staticskytool.h (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/staticskytool.h (revision 33415) @@ -34,5 +34,13 @@ STATICSKYTOOL_MODE_UPDATERESULT, STATICSKYTOOL_MODE_EXPORTRUN, - STATICSKYTOOL_MODE_IMPORTRUN + STATICSKYTOOL_MODE_IMPORTRUN, + + STATICSKYTOOL_MODE_DEFINESKYCALRUN, + STATICSKYTOOL_MODE_PENDINGSKYCALRUN, + STATICSKYTOOL_MODE_UPDATESKYCALRUN, + STATICSKYTOOL_MODE_ADDSKYCALRESULT, + STATICSKYTOOL_MODE_UPDATESKYCALRESULT, + STATICSKYTOOL_MODE_SKYCALRESULT, + STATICSKYTOOL_MODE_REVERTSKYCALRESULT, } staticskytoolMode; Index: branches/meh_branches/ppstack_test/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/staticskytoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/staticskytoolConfig.c (revision 33415) @@ -48,4 +48,5 @@ // XXX need a 'ra_min,max', 'dec_min,max' : to do this, we need a table of skycell boundaries psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); @@ -107,8 +108,9 @@ psMetadata *resultArgs= psMetadataAlloc(); psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); - // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); - // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -138,5 +140,102 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - + + // -defineskycalrun + psMetadata *defineskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_sky_id", 0, "search for sky_id", 0); + psMetadataAddS64(defineskycalrunArgs, PS_LIST_TAIL, "-select_stack_id", 0, "search for stack_id", 0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_tess_id", 0, "search for tess_id", NULL); + psMetadataAddF32(defineskycalrunArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddTime(defineskycalrunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-rerun", 0, "queue new run even if one already exists for inputs", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-check_inputs", 0, "list inputs, do not modify database", false); + psMetadataAddBool(defineskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updateskycalrun + psMetadata *updateskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by stack ID", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + + // -pendingskycalrun + psMetadata *pendingskycalrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(pendingskycalrunArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddStr(pendingskycalrunArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddU64(pendingskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingskycalrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + + // -addskycalresult + psMetadata *addskycalresultArgs = psMetadataAlloc(); + psMetadataAddS64(addskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define camtool ID (required)", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_ra", 0, "define exposure E ra", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-sigma_dec", 0, "define exposure E dec", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_obs", 0, "define observed zero point", 0); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-zpt_stdev", 0, "define observed zero point stdandard deviation", 0); + + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); + psMetadataAddF32(addskycalresultArgs, PS_LIST_TAIL, "-dtime_astrom", 0, "define elapsed time for astrometry (seconds)", NAN); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + // psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + psMetadataAddS32(addskycalresultArgs, PS_LIST_TAIL, "-n_astrom", 0, "define number of astrometry reference objects", 0); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16(addskycalresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psphot", 0, "define psphot version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_psastro", 0, "define psastro version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL); + psMetadataAddStr(addskycalresultArgs, PS_LIST_TAIL, "-ver_streaks", 0, "define streaksremove version", NULL); + + // -revertskycal + psMetadata *revertskycalArgs= psMetadataAlloc(); + psMetadataAddS64(revertskycalArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by staticsky ID", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", 0); + psMetadataAddStr(revertskycalArgs, PS_LIST_TAIL, "-filter", PS_META_DUPLICATE_OK, "search by filter", 0); + psMetadataAddS16(revertskycalArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertskycalArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false); + + // -updateskycalresult + psMetadata *updateskycalresultArgs = psMetadataAlloc(); + psMetadataAddS64(updateskycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code (required)", 0); + psMetadataAddS16(updateskycalresultArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality code", 0); + + // -skycalresult + psMetadata *skycalresultArgs= psMetadataAlloc(); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-skycal_id", 0, "search by skycal ID", 0); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0); + psMetadataAddS64(skycalresultArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", 0); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddStr(skycalresultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddS16(skycalresultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddU64(skycalresultArgs, PS_LIST_TAIL, "-limit", 0, "limit skycalresult set to N items", 0); + psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -155,4 +254,11 @@ PXOPT_ADD_MODE("-exportrun", "Export run", STATICSKYTOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "Import run", STATICSKYTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-defineskycalrun", "Define a new skycalrun", STATICSKYTOOL_MODE_DEFINESKYCALRUN, defineskycalrunArgs); + PXOPT_ADD_MODE("-updateskycalrun", "Update a skycalrun", STATICSKYTOOL_MODE_UPDATESKYCALRUN, updateskycalrunArgs); + PXOPT_ADD_MODE("-pendingskycalrun", "Get skcal runs to do", STATICSKYTOOL_MODE_PENDINGSKYCALRUN, pendingskycalrunArgs); + PXOPT_ADD_MODE("-addskycalresult", "add skycal result", STATICSKYTOOL_MODE_ADDSKYCALRESULT, addskycalresultArgs); + PXOPT_ADD_MODE("-revertskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_REVERTSKYCALRESULT, revertskycalArgs); + PXOPT_ADD_MODE("-updateskycal", "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs); + PXOPT_ADD_MODE("-skycalresult", "Get result of skycal run", STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/meh_branches/ppstack_test/ippTools/src/vptool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/vptool.c (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/vptool.c (revision 33415) @@ -0,0 +1,1849 @@ +/* + * vptool.c + * + * Copyright (C) 2011 IfA University of Hawaii + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "pxtools.h" +#include "pxdata.h" +#include "pxchip.h" + +#include "vptool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool pendingrunMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool revertrunMode(pxConfig *config); +static bool listrunMode(pxConfig *config); +static bool pendingimfileMode(pxConfig *config); +static bool addprocessedcellMode(pxConfig *config); +static bool processedcellMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) { + psLibInit(NULL); + + pxConfig *config = vptoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(VPTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(VPTOOL_MODE_PENDINGRUN, pendingrunMode); + MODECASE(VPTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(VPTOOL_MODE_REVERTRUN, revertrunMode); + MODECASE(VPTOOL_MODE_LISTRUN, listrunMode); + MODECASE(VPTOOL_MODE_PENDINGIMFILE, pendingimfileMode); + MODECASE(VPTOOL_MODE_ADDPROCESSEDCELL, addprocessedcellMode); + MODECASE(VPTOOL_MODE_PROCESSEDCELL, processedcellMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + +// Queue exposures for videophot processing +static bool queue_exposures(pxConfig *config, // Configuration + const psArray *exps, // Exposures information + const char *label, // Label, or NULL to inherit + const char *data_group, // Data group, or NULL to inherit + const char *workdir, // Working directory, or NULL to inherit + const char *note, // Note + const psS64 dest_id + ) +{ + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + +# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) \ + PTYPE VALUE; \ + { \ + bool status; \ + VALUE = psMetadataLookup##CTYPE(&status, md, NAME); \ + if (!status) { \ + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \ + return false; \ + } \ + } + + // loop over our list of exp_ids + for (long i = 0; i < psArrayLength(exps); i++) { + psMetadata *md = exps->data[i]; + + rawExpRow *row = rawExpObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + return false; + } + + GET_VALUE (psS64, S64, exp_id, "exp_id"); + GET_VALUE (psString, Str, raw_workdir, "workdir"); + GET_VALUE (psString, Str, raw_label, "label"); + GET_VALUE (psString, Str, raw_reduction, "reduction"); + // GET_VALUE (psString, Str, raw_expgroup, "expgroup"); + GET_VALUE (psString, Str, raw_dvodb, "dvodb"); + GET_VALUE (psString, Str, raw_tess_id, "tess_id"); + GET_VALUE (psString, Str, raw_end_stage, "end_stage"); + + if (!row->exp_id) { + psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id"); + return false; + } + + // queue the exp + if (!vpRunInsert(config->dbh, + 0, //vp_id + exp_id, + "new", + label ? label : raw_label, + data_group ? data_group : (label ? label : raw_label), + workdir ? workdir : raw_workdir, + note, + dest_id, + NULL, // outroot + 0, // dtime_script + NULL, // hostname + 0 // fault + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue exp_id: %" PRId64, exp_id); + return false; + } + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_S64(dest_id, config->args, "-set_dest_id", false, false); + + // default + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "="); // define using newExp label + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("vptool_find_rawexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!rerun) { + psStringAppend(&query, "\n AND vp_id IS NULL\n AND (vpRun.label = '%s' OR vpRun.label IS NULL)", label); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!queue_exposures(config, output, label, data_group, workdir, note, dest_id)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for videophot processing."); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool pendingrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + // Do we need pxchipArgs? + pxchipGetSearchArgs (config, where); //chipRun, rawExp + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "=="); + + psString query = pxDataGet("vptool_pendingrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + psStringAppend(&query, "\nORDER BY priority DESC, vp_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "vpPendingRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +#ifdef notdef +static bool definecopyMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE"); + + // psListLength(where->list) is at least 1 because exp_type defaults to "object" + // so we require a list longer than 1 entry + if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + // default + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find the exp_id of all the exposures that we want to queue up. + psString query = pxDataGet("vptool_definecopy.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + + psFree(whereClause); + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + if (!queue_exposures(config, output, workdir, label, data_group, dist_group, + note)) { + psError(psErrorCodeLast(), false, "Unable to queue exposures for chip."); + psFree(output); + return false; + } + psFree(output); + + return true; +} +#endif + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); +// pxchipGetSearchArgs (config, where); // rawExp, vpRun + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "vpRun.label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "vpRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "vpRun.data_group", "LIKE"); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-set_hostname", false, false); + PXOPT_LOOKUP_STR(outroot, config->args, "-set_outroot", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-set_dtime_script", false, false); + + if (!state && !label && !data_group && !note && !hostname && !fault) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "must supply at least one set parameter"); + return false; + } + + psString query = psStringCopy("UPDATE vpRun"); + + char *separator = " SET "; + char * comma = ","; + +# define addColumn(_tab, _fmt, _val) \ + do { \ + if (_val) { \ + psStringAppend(&query, "%s %s.%s =" _fmt, separator, _tab, #_val, _val); \ + separator = comma; \ + } \ + } while (0) + + addColumn("vpRun", "'\%s'", state); + addColumn("vpRun", "'\%s'", label); + addColumn("vpRun", "'\%s'", data_group); + addColumn("vpRun", "'\%s'", note); + addColumn("vpRun", "'\%s'", hostname); + addColumn("vpRun", "'\%s'", outroot); + addColumn("vpRun", "%d", fault); + addColumn("vpRun", "%f", dtime_script); + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + psStringAppend(&query, "\nWHERE %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psFree(query); + + return true; +} + + +static bool pendingimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + // Do we need pxchipArgs? + pxchipGetSearchArgs (config, where); //chipRun, rawExp + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "=="); + + psString query = pxDataGet("vptool_pendingimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "vpPendingImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addprocessedcellMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // vp_id, class_id, and cell_id are required + PXOPT_LOOKUP_S64(vp_id, config->args, "-vp_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(cell_id, config->args, "-cell_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_photom, config->args, "-dtime_photom", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + // default values + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!vpProcessedCellInsert(config->dbh, + vp_id, + class_id, + cell_id, + dtime_photom, + quality, + path_base, + fault + )) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool processedcellMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-vp_id", "vp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-cell_id", "cell_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "vpRun.data_group", "LIKE"); + + psString query = pxDataGet("vptool_processedcell.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "vpProcessedCell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool revertrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-vp_id", "vpRun.vp_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "vpRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "vpProcessedCell.fault", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + // find vpRuns with faulted cells + psString query = pxDataGet("vptool_revertrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + int numUpdated = psDBAffectedRows(config->dbh); + psLogMsg("vptool", PS_LOG_INFO, "Reverted %d vpRuns", numUpdated); + + return true; +} + +#ifdef NOTDEF +static bool revertcleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + PXOPT_LOOKUP_STR(state, config->args, "-state", false, false); + + char* newState = NULL; + if (!state) { + state = "error_cleaned"; + } + if (!strcmp(state, "error_cleaned")) { + newState = "goto_cleaned"; + } else if (!strcmp(state, "error_purged")) { + newState = "goto_purged"; + } else if (!strcmp(state, "error_scrubbed")) { + newState = "goto_scrubbed"; + } else { + psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed"); + return false; + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("vptool_revertcleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + return true; +} + +static bool updateprocessedcellMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + if (psListLength(where->list) == 0) { + psError(PS_ERR_UNKNOWN, true, "search parameters are required"); + return false; + } + + if (!state) { + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + return false; + } + psFree(where); + } + else { + if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); + return(false); + } + } + + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!chipMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM chipMask"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +// return the list of labels which are NOT blocked +static bool unmaskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + if (where->list->n < 1) { + psFree(where); + where = NULL; + } + + psString query = pxDataGet("vptool_unmasked.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, "chipUnmask"); + psFree(where); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipUnmask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM chipMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("vptool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool pendingcleanupimfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (chip_id) { + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("vptool_pendingcleanupimfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipPendingCleanupImfile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("vptool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool runMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // make sure that the state string is valid + if (!pxIsValidState(state)) { + psError(PXTOOLS_ERR_CONFIG, false, "%s is not a valid state", state); + return false; + } + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + psString query = pxDataGet("vptool_run.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool advanceexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + // look for completed chipPendingExp + // migrate them to chipProccessedExp & camPendingExp + psString query = pxDataGet("vptool_completely_processed_exp.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 chip_id = psMetadataLookupS64(NULL,row,"chip_id"); + + psString software_ver = NULL; + psS64 maskfrac_npix = 0; + psF32 maskfrac_static = 0; + psF32 maskfrac_dynamic = 0; + psF32 maskfrac_magic = 0; + psF32 maskfrac_advisory = 0; + + // Calculate run level masking and software state + if (!pxCoalesceRunStatus(config,"vptool_coalesce_run.sql",chip_id, + &software_ver,&maskfrac_npix, + &maskfrac_static,&maskfrac_dynamic, + &maskfrac_magic,&maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + psMetadataAddStr(row, PS_LIST_TAIL, "software_ver", 0, "Software version", software_ver); + psMetadataAddS32(row, PS_LIST_TAIL, "maskfrac_npix", 0, "Number of pixels used for mask stats", (psS32) maskfrac_npix); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_static", 0, "Fraction of static masked pixels", maskfrac_static); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_dynamic", 0, "Fraction of dynamic masked pixels", maskfrac_dynamic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_magic", 0, "Fraction of magic masked pixels", maskfrac_magic); + psMetadataAddF32(row, PS_LIST_TAIL, "maskfrac_advisory", 0, "Fraction of advisory pixels", maskfrac_advisory); +/* psWarning("ADVANCE %ld %s %d %f %f %f %f\n",chip_id,software_ver,maskfrac_npix,maskfrac_static,maskfrac_dynamic,maskfrac_magic,maskfrac_advisory); */ + chipRunRow *chipRun = chipRunObjectFromMetadata(row); + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // Set chipRun.software_ver to the appropriate value + if (software_ver) { + if (!pxSetRunSoftware(config, "chipRun", "chip_id", chip_id, software_ver)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + // Set chipRun.maskfrac* to the appropriate values. + if (maskfrac_npix) { + if (!pxSetRunMaskfrac(config, "chipRun", "chip_id",chip_id, maskfrac_npix, maskfrac_static, + maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipRun.software_ver for chip_id: %" PRId64, + chip_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return(false); + } + } + // set chipRun.state to 'stop' and update the magicked state + if (!pxchipRunSetState(config, chipRun->chip_id, "full", chipRun->magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id); + psFree(chipRun); + psFree(output); + return false; + } + + // should we stop here or proceed on to the cam stage? + // NULL for end_stage means go as far as possible + if (chipRun->end_stage && psStrcasestr(chipRun->end_stage, "chip")) { + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(chipRun); + continue; + } + // else continue on... + + // camQueueChipID() can only be run after chipRun.state has been set to + // stop + if (!pxcamQueueByChipID(config, + chipRun->chip_id, + chipRun->workdir, + chipRun->label, + chipRun->data_group, + chipRun->dist_group, + chipRun->reduction, + chipRun->expgroup, + chipRun->dvodb, + chipRun->tess_id, + chipRun->end_stage, + chipRun->magicked, + NULL // note does not propagate + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to queue camPendingExp"); + psFree(chipRun); + psFree(output); + return false; + } + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(chipRun); + } + + psFree(output); + + return true; +} + +// update chipProcessedImfile.data_state to given value. +// afterwards, if all imfiles in the exposure have the new state, update the state for the exposure as well +// shared code for the modes -tocleanedimfile -tofullimfile -topurgedimfile + +static bool change_imfile_data_state(pxConfig *config, psString data_state, psString run_state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // chip_id, class_id are required + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + + psString query = pxDataGet("vptool_change_imfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psString set_magicked_imfile = psStringCopy(""); + psString set_magicked_run = psStringCopy(""); + if (!strcmp(data_state, "full")) { + // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will + // block warp processing until destreaking has been done + // otherwise copy magicked from the rawImfile + // Same thing for chipRun/rawExp + psStringAppend(&set_magicked_imfile, "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0" + " AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)"); + psStringAppend(&set_magicked_run, "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), " + " chipRun.magicked, rawExp.magicked)"); + + } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { + // if magicked is non-zero set it to -1 + // Once one imfile has been cleaned, the chipRun is no longer 'magicked' + psStringAppend(&set_magicked_imfile, "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1)," + " chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)"); + } + + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + psFree(set_magicked_imfile); + psFree(set_magicked_run); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row"); + return false; + } + + query = pxDataGet("vptool_change_exp_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) { + psFree(query); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} +static bool tocleanedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "cleaned", "goto_cleaned"); +} +static bool tofullimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "full", "update"); +} +static bool topurgedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 3; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(dummy, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); + + ExportTable tables [] = { + {"chipRun", "vptool_export_run.sql"}, + {"chipImfile", "vptool_export_imfile.sql"}, + {"chipProcessedImfile", "vptool_export_processed_imfile.sql"}, + }; + + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + bool success = true; + if (!strcmp(tables[i].tableName, "chipRun")) { + success = pxSetStateCleaned("chipRun", "state", output); + } else if (!strcmp(tables[i].tableName, "chipProcessedImfile")) { + success = pxSetStateCleaned("chipProcessedImfile", "data_state", output); + } + if (!success) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + unsigned int nFail = 0; // Number of failed lines + psMetadata *input = psMetadataConfigRead(NULL, &nFail, infile, false); + if (nFail) { + psError(PS_ERR_IO, false, "%d failed lines in input", nFail); + psFree(input); + return false; + } + + psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted + + psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + // first item is the dbversion, skip it + psMetadataItem *dbversion = psMetadataGetAndIncrement(iter); + (void) dbversion; + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + PXMIRROR_PRIMARY(item, "chipRun", chipRunRow, chipRunObjectFromMetadata, identifiers, chip_id, + chipRunInsertObject, config->dbh, + { psFree(iter); psFree(identifiers); psFree(input); }); + + PXMIRROR_OTHER(item, "chipImfile", chipImfileRow, chipImfileObjectFromMetadata, identifiers, chip_id, + chipImfileInsertObject, config->dbh, + { psFree(iter); psFree(identifiers); psFree(input); }); + + PXMIRROR_OTHER(item, "chipProcessedImfile", chipProcessedImfileRow, + chipProcessedImfileObjectFromMetadata, identifiers, chip_id, + chipProcessedImfileInsertObject, config->dbh, + { psFree(iter); psFree(identifiers); psFree(input); }); + } + psFree(iter); + psFree(input); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psLogMsg("vptool", PS_LOG_INFO, "%ld chipRuns added", identifiers->n); + psFree(identifiers); + + return true; +} + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("vptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + +// a very specfic function to queue a cleaned chipProcessedImfile to be updated +static bool setimfiletoupdateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + psString query = pxDataGet("vptool_setimfiletoupdate.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + psString setHook = psStringCopy(""); + if (label) { + psStringAppend(&setHook, "\n , chipRun.label = '%s'", label); + } + + if (class_id) { + psStringAppend(&query, " AND (chipProcessedImfile.class_id = '%s')", class_id); + } + + // we do not update components with the magic fault value. They are non-updateable + psStringAppend(&query, " AND (chipProcessedImfile.fault != %d)", PXTOOL_DO_NOT_REVERT_FAULT); + + if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(setHook); + psFree(query); + + return true; +} +#endif // NOTDEF +static bool listrunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE"); + pxAddLabelSearchArgs (config, where, "-dist_group", "chipRun.dist_group", "LIKE"); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipRun"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); + return false; + } + + psString query = pxDataGet("vptool_listrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : ""); + psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || where2) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required"); + return false; + } + psFree(where); + + if (pstamp_order) { + // put runs in order of exposure id with newest chip Runs first + // The postage stamp parser depends on this behavior + psStringAppend(&query, "\nORDER by exp_id, chip_id DESC"); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("vptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/meh_branches/ppstack_test/ippTools/src/vptool.h =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/vptool.h (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/vptool.h (revision 33415) @@ -0,0 +1,39 @@ +/* + * vptool.h + * + * Copyright (C) 2011 University of Hawaii IFA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef VPTOOL_H +#define VPTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + VPTOOL_MODE_NONE = 0x0, + VPTOOL_MODE_DEFINEBYQUERY, + VPTOOL_MODE_PENDINGRUN, + VPTOOL_MODE_UPDATERUN, + VPTOOL_MODE_REVERTRUN, + VPTOOL_MODE_LISTRUN, + VPTOOL_MODE_PENDINGIMFILE, + VPTOOL_MODE_ADDPROCESSEDCELL, + VPTOOL_MODE_PROCESSEDCELL, +} vptoolMode; + +pxConfig *vptoolConfig(pxConfig *config, int argc, char **argv); + +#endif // VPTOOL_H Index: branches/meh_branches/ppstack_test/ippTools/src/vptoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/vptoolConfig.c (revision 33415) +++ branches/meh_branches/ppstack_test/ippTools/src/vptoolConfig.c (revision 33415) @@ -0,0 +1,297 @@ +/* + * vptoolConfig.c + * + * Copyright (C) 2006-2008 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include + +#include + +#include "pxtools.h" +#include "vptool.h" + +pxConfig *vptoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + config->modules = pmConfigRead(&argc, argv, NULL); + if (! config->modules) { + psError(psErrorCodeLast(), false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + psMetadata *definebyqueryArgs = psMetadataAlloc(); + // allow selecting by all of the args used to queue chipRuns + pxchipSetSearchArgs (definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by newExp label (LIKE comparison)", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-set_dest_id", 0, "define destination", 0); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue exposures even if a previous vpRun exists", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -pendingrun + psMetadata *pendingrunArgs = psMetadataAlloc(); + psMetadataAddS64(pendingrunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + pxchipSetSearchArgs(pendingrunArgs); + psMetadataAddU64(pendingrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -updaterun + psMetadata *updaterunArgs = psMetadataAlloc(); +// pxchipSetSearchArgs (updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by vpRun ID", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label (LIKE comparison)", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); + psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-set_dtime_script", 0, "define elapsed time for script (seconds)", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_hostname", 0, "set hostname", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_outroot", 0, "set outroot", NULL); + psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0); + + // -revertrun + psMetadata *revertrunArgs = psMetadataAlloc(); + psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + // pxchipSetSearchArgs(revertrunArgs); + psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + // -pendingimfile + psMetadata *pendingimfileArgs = psMetadataAlloc(); + psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0); + pxchipSetSearchArgs(pendingimfileArgs); + psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -addprocessedcell + psMetadata *addprocessedcellArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedcellArgs, PS_LIST_TAIL, "-vp_id", 0, "define chip ID (required)", 0); + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-class_id", 0, "define class ID (required)", NULL); + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-cell_id", 0, "define class ID (required)", NULL); + + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location (required)", NULL); + psMetadataAddF32(addprocessedcellArgs, PS_LIST_TAIL, "-dtime_photom", 0, "define elapsed time for photometry (seconds)", NAN); + psMetadataAddStr(addprocessedcellArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + + psMetadataAddS16(addprocessedcellArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); + psMetadataAddS16(addprocessedcellArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0); + + + // -processedcell + psMetadata *processedcellArgs = psMetadataAlloc(); + pxchipSetSearchArgs(processedcellArgs); + psMetadataAddS64(processedcellArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-cell_id", 0, "search by class ID", NULL); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(processedcellArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + + psMetadataAddU64(processedcellArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedcellArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + pxspaceAddArguments(processedcellArgs); + +#ifdef notdef + // -updateprocessedcell + psMetadata *updateprocessedcellArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedcellArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(updateprocessedcellArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); + psMetadataAddS16(updateprocessedcellArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + psMetadataAddStr(updateprocessedcellArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddS16(updateprocessedcellArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); +#endif // notdef + // -listrun + psMetadata *listrunArgs = psMetadataAlloc(); + pxchipSetSearchArgs(listrunArgs); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-vp_id", 0, "search by vpRun ID", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by vpRun state", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by vpRun dist_group (LIKE comparison)", NULL); + // pxmagicAddArguments(listrunArgs); + + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-pstamp_order", 0, "order results for postage stamp server", false); + psMetadataAddU64(listrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); + psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + pxspaceAddArguments(listrunArgs); + +#ifdef notdef + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddStr(maskedArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(maskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unmasked + psMetadata *unmaskedArgs = psMetadataAlloc(); + psMetadataAddStr(unmaskedArgs, PS_LIST_TAIL, "-label", 0, "restrict to specified label", NULL); + psMetadataAddBool(unmaskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(unmaskedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupimfile + psMetadata *pendingcleanupimfileArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -revertcleanup + psMetadata *revertcleanupArgs = psMetadataAlloc(); + psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by vpRun label (LIKE comparison)", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by vpRun data_group (LIKE comparison)", NULL); + psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0, "search by current state", NULL); + psMetadataAddS16(revertcleanupArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + // -run + psMetadata *runArgs = psMetadataAlloc(); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + psMetadataAddBool(runArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + pxchipSetSearchArgs(runArgs); + psMetadataAddS64(runArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); + + // -advanceexp + psMetadata *advanceexpArgs = psMetadataAlloc(); + psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "advance exposures for specified label", NULL); + psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -tocleanedimfile + psMetadata *tocleanedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(tocleanedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -tofullimfile + psMetadata *tofullimfileArgs = psMetadataAlloc(); + psMetadataAddS64(tofullimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(tofullimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -topurgedimfile + psMetadata *topurgedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-vp_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + // pxchipSetSearchArgs (exportrunArgs); XXX include search terms? + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-vp_id", 0, "export this chip ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-vp_id", 0, "search by chip ID", 0); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +#endif //notdef + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from raw exposures", VPTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-pendingrun", "list runs ready to process", VPTOOL_MODE_PENDINGRUN, pendingrunArgs); + PXOPT_ADD_MODE("-updaterun", "change vpRun properties", VPTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-revertrun","clear a faulted vpRun", VPTOOL_MODE_REVERTRUN, revertrunArgs); + PXOPT_ADD_MODE("-pendingimfile", "list pending imfiles", VPTOOL_MODE_PENDINGIMFILE, pendingimfileArgs); + + PXOPT_ADD_MODE("-addprocessedcell", "add a processed cell", VPTOOL_MODE_ADDPROCESSEDCELL, addprocessedcellArgs); + PXOPT_ADD_MODE("-processedcell", "show processed cells", VPTOOL_MODE_PROCESSEDCELL, processedcellArgs); + PXOPT_ADD_MODE("-listrun", "list vpRuns", VPTOOL_MODE_LISTRUN, listrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/meh_branches/ppstack_test/ippTools/src/warptool.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/warptool.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/warptool.c (revision 33415) @@ -975,4 +975,5 @@ psFree(where); + psArray *allOutput = psArrayAllocEmpty(0); // Now loop over the labels and query each individually for (long i = 0; i < psArrayLength(labelOutput); i++) { @@ -983,5 +984,5 @@ psString query = psStringCopy(originalQuery); - + psStringAppend(&query,"\nORDER BY warp_id"); // treat limit == 0 as "no limit" psString limitString = psStringCopy(""); @@ -992,4 +993,5 @@ psStringAppend(&query, "%s", limitString); } + psString thisWhere = NULL; if (whereStr) { @@ -1023,9 +1025,11 @@ long outputLength = psArrayLength(output); if (outputLength) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; + for (int i = 0; i< outputLength; i++) { + psPtr ptr = output->data[i]; + allOutput = psArrayAdd(allOutput, outputLength - i + 10, ptr); + if (!allOutput) { + psError(PS_ERR_UNKNOWN, false, "failed to add element to array"); + return false; + } } } else { @@ -1044,4 +1048,13 @@ } } + long allLabelsLength = psArrayLength(allOutput); + if (allLabelsLength) { + if (!ippdbPrintMetadatas(stdout, allOutput, "warpPendingSkyCell", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(allOutput); + return false; + } + } + psFree(allOutput); psFree(labelOutput); psFree(whereStr); @@ -2039,8 +2052,12 @@ // magicked is only an argument for for -tofullskyfile PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); if (magicked) { psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); } + if (quality) { + psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.quality = %"PRId16, quality); + } } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { // if magicked is currently nonzero set it to -1 @@ -2421,4 +2438,6 @@ PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); @@ -2562,4 +2581,13 @@ psFree(query); - return true; -} + if (!skycell_id) { + // If we are updateing a whole warpRun set skycells with bad quality to 'full' + query = "UPDATE warpSkyfile SET data_state ='full', fault = 0 WHERE warp_id = %" PRId64 " AND quality != 0 AND (data_state != 'full')"; + if (!p_psDBRunQueryF(config->dbh, query, warp_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + + return true; +} Index: branches/meh_branches/ppstack_test/ippTools/src/warptoolConfig.c =================================================================== --- branches/meh_branches/ppstack_test/ippTools/src/warptoolConfig.c (revision 32161) +++ branches/meh_branches/ppstack_test/ippTools/src/warptoolConfig.c (revision 33415) @@ -269,7 +269,9 @@ psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); - psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); - psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure id", 0); + psMetadataAddStr(listrunArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", NULL); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip id", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam id", 0); + psMetadataAddS64(listrunArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag (fake_id)", 0); psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); psMetadataAddTime(listrunArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); @@ -396,5 +398,6 @@ psMetadataAddStr(tofullskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS64(tofullskyfileArgs, PS_LIST_TAIL, "-set_magicked", 0, "new value for magicked", 0); - + psMetadataAddS16(tofullskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "new value for quality", 0); + // -updateskyfile psMetadata *updateskyfileArgs = psMetadataAlloc(); Index: branches/meh_branches/ppstack_test/ippconfig/Makefile.am =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/Makefile.am (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/Makefile.am (revision 33415) @@ -18,4 +18,5 @@ lbc_red \ lulin \ + ssp \ uh8k Index: branches/meh_branches/ppstack_test/ippconfig/configure.ac =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/configure.ac (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/configure.ac (revision 33415) @@ -29,4 +29,5 @@ lbc_red/Makefile lulin/Makefile + ssp/Makefile uh8k/Makefile ]) Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20090120.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20090120.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20090120.config (revision 33415) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -505,5 +505,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20090220.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20090220.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20090220.config (revision 33415) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -505,5 +505,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END @@ -590,4 +590,6 @@ XY76 STR 1111111111111111111111111111111111111111111111111111111111111111 END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction XY01 BOOL FALSE Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20091209.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20091209.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20091209.config (revision 33415) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -505,5 +505,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END @@ -525,7 +525,135 @@ # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +# PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 +# XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 BOOL FALSE +# XY25 BOOL FALSE +# XY26 BOOL FALSE +# XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 +# XY30 BOOL FALSE +# XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 +# XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY33 BOOL FALSE +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# XY36 BOOL FALSE +# XY37 BOOL FALSE +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 BOOL FALSE +# XY44 BOOL FALSE +# XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY46 BOOL FALSE +# XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 +# XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 +# XY61 BOOL FALSE +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 BOOL FALSE +# XY73 BOOL FALSE +# XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 +# XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 +# XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY35 BOOL FALSE +# XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 +# XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 +# XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END +# PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -536,5 +664,5 @@ XY12 BOOL FALSE XY13 BOOL FALSE - XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 + XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 XY16 BOOL FALSE @@ -578,5 +706,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -588,65 +717,82 @@ END -# PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100122.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100122.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100122.config (revision 33415) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -506,5 +506,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END @@ -527,8 +527,136 @@ # Pattern correction information: +# # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +# PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 +# XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 BOOL FALSE +# XY25 BOOL FALSE +# XY26 BOOL FALSE +# XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 +# XY30 BOOL FALSE +# XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 +# XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY33 BOOL FALSE +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# XY36 BOOL FALSE +# XY37 BOOL FALSE +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 BOOL FALSE +# XY44 BOOL FALSE +# XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY46 BOOL FALSE +# XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 +# XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 +# XY61 BOOL FALSE +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 BOOL FALSE +# XY73 BOOL FALSE +# XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 +# XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 +# XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY35 BOOL FALSE +# XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 +# XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 +# XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -581,5 +709,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -591,65 +720,82 @@ END -# PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100228.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100228.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100228.config (revision 33415) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -506,5 +506,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END @@ -526,8 +526,136 @@ # Pattern correction information: +# # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? +# PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction +# XY01 BOOL FALSE +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 STR 0011100100000000000000000000000000000000000000000000000000000000 +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000000100000000000000000000000000000000000000000000 # xy23 +# XY15 STR 1111111100000000000000000000000000000000000000000000000000000000 # cols: 0 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 BOOL FALSE +# XY25 BOOL FALSE +# XY26 BOOL FALSE +# XY27 STR 1111111111111111111111111111111100000000000000000000000011111111 # cols: 0,1,2,3,7 +# XY30 BOOL FALSE +# XY31 STR 0000000000000000000000000000000000000000000000000000000011111111 # cols: 7 +# XY32 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY33 BOOL FALSE +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# XY36 BOOL FALSE +# XY37 BOOL FALSE +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 BOOL FALSE +# XY44 BOOL FALSE +# XY45 STR 0000000000000000000000001111111100000000000000000000000011111111 # cols: 3,7 +# XY46 BOOL FALSE +# XY47 STR 1111111100000000000000001111111100000000111111110000000011111111 # cols: 0,3,5,7 +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 STR 1111111111111111111111110000000000000000000000001111111111111111 # cols: 0,1,2,6,7 +# XY60 STR 0000000000000000000000000000000000000000000001000000000000000000 # cell xy55 +# XY61 BOOL FALSE +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 BOOL FALSE +# XY73 BOOL FALSE +# XY74 STR 0000000000000000111111110000000000000000000000000000000011111111 # cols: 2,7 +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END + +# # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 +# XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 +# XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY35 BOOL FALSE +# XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 +# XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 +# XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY41 BOOL FALSE +# XY42 BOOL FALSE +# XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END # PATTERN.ROW BOOL TRUE # Do any row pattern correction at all? PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -580,5 +708,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -590,65 +719,82 @@ END +PATTERN.CELL BOOL FALSE +PATTERN.CONTINUITY BOOL TRUE + # PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100723.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100723.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/format_20100723.config (revision 33415) @@ -185,5 +185,5 @@ FPA.ALT STR ALT FPA.AZ STR AZ - # FPA.TEMP STR DETTEM + FPA.TEMP STR DETTEM FPA.M1X STR M1X FPA.M1Y STR M1Y @@ -506,5 +506,5 @@ DATABASE METADATA # this rule is fragile : does not match the camera - FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 +# FPA.TEMP STR SELECT ccd_temp FROM rawExp WHERE dateobs >= '{FPA.DATE}' and abs(timediff(dateobs, cast('{FPA.TIME}' as datetime))) < 2 END @@ -529,5 +529,6 @@ PATTERN.ROW.SUBSET METADATA # List of chips and whether to do row pattern correction XY01 BOOL FALSE - XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 +# XY02 STR 0000000000000000000000000000000000000000000000000000000000000001 # xy77 + XY02 BOOL FALSE XY03 BOOL FALSE XY04 BOOL FALSE @@ -580,5 +581,6 @@ XY64 BOOL FALSE XY65 BOOL FALSE - XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 +# XY66 STR 0000000000000000000000000000000000000000000000000000000000000100 # cell xy75 + XY66 BOOL FALSE XY67 BOOL FALSE XY71 BOOL FALSE @@ -590,65 +592,80 @@ END -# PATTERN.CELL BOOL TRUE # by default, do not do any cell pattern correction -PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction - XY01 BOOL FALSE - XY02 BOOL FALSE - XY03 BOOL FALSE - XY04 BOOL FALSE - XY05 BOOL FALSE - XY06 STR 1101000000010000000110100100000000000000000000000000000000000000 - XY10 BOOL FALSE - XY11 BOOL FALSE - XY12 BOOL FALSE - XY13 BOOL FALSE - XY14 STR 0000000000000000111111110000000000000000000000010000000000101101 - XY15 STR 0011111100011000000001000000010000001001000111110001010000011111 - XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 - XY17 BOOL FALSE - XY20 BOOL FALSE - XY21 BOOL FALSE - XY22 BOOL FALSE - XY23 BOOL FALSE - XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 - XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 - XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 - XY27 BOOL FALSE - XY30 BOOL FALSE - XY31 BOOL FALSE - XY32 BOOL FALSE - XY33 BOOL FALSE - XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 - XY35 BOOL FALSE - XY36 STR 0000000000000000000000000000000000010000000000000000000000000000 - XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 - XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 - XY41 BOOL FALSE - XY42 BOOL FALSE - XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 - XY44 BOOL FALSE - XY45 BOOL FALSE - XY46 BOOL FALSE - XY47 BOOL FALSE - XY50 BOOL FALSE - XY51 BOOL FALSE - XY52 BOOL FALSE - XY53 BOOL FALSE - XY54 BOOL FALSE - XY55 BOOL FALSE - XY56 BOOL FALSE - XY57 BOOL FALSE - XY60 BOOL FALSE - XY61 BOOL FALSE - XY62 STR 0000000000000000000000000000000000001000000000000000000100001010 - XY63 BOOL FALSE - XY64 BOOL FALSE - XY65 BOOL FALSE - XY66 BOOL FALSE - XY67 BOOL FALSE - XY71 BOOL FALSE - XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 - XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 - XY74 BOOL FALSE - XY75 BOOL FALSE - XY76 BOOL FALSE -END +PATTERN.CONTINUITY BOOL TRUE + PATTERN.CELL BOOL FALSE # by default, do not do any cell pattern correction +# PATTERN.CELL.SUBSET METADATA # List of chips and whether to do cell pattern correction +# XY01 BOOL FALSE +# XY02 BOOL FALSE +# XY03 BOOL FALSE +# XY04 BOOL FALSE +# XY05 BOOL FALSE +# # XY06 STR 0101000000010000000110000100000000000000000000000000000000000000 +# XY06 BOOL FALSE +# XY10 BOOL FALSE +# XY11 BOOL FALSE +# XY12 BOOL FALSE +# XY13 BOOL FALSE +# XY14 STR 0000000000000000111111110000000000000000000000000000000000000000 +# XY15 STR 1111111110000000100000000000000000000000000000000000000000000000 +# # XY16 STR 0000000100000000000000000000000000000001000000000000000000000000 +# XY16 BOOL FALSE +# XY17 BOOL FALSE +# XY20 BOOL FALSE +# XY21 BOOL FALSE +# XY22 BOOL FALSE +# XY23 BOOL FALSE +# # XY24 STR 0000000000100000000000000110100000100000001000000011100000000000 +# XY24 BOOL FALSE +# # XY25 STR 0100010000001001001011000000001000000111000110000000100000000010 +# XY25 BOOL FALSE +# # XY26 STR 0000000000000100000000000000000000001000001000000000000000000000 +# XY26 BOOL FALSE +# XY27 BOOL FALSE +# XY30 BOOL FALSE +# XY31 BOOL FALSE +# XY32 BOOL FALSE +# XY33 BOOL FALSE +# # XY34 STR 0000000000000000000000000000000000000001000011000000101000111111 +# XY34 BOOL FALSE +# XY35 BOOL FALSE +# # XY36 STR 0001000000000000000000000000000000010000000000000000000000000000 +# XY36 BOOL FALSE +# # XY37 STR 0000000000000000000000000000000100010001000000001011000010110110 ? +# XY37 BOOL FALSE +# # XY40 STR 0000000000000000000000000010110000000000000001110000001000011111 +# XY40 BOOL FALSE +# XY41 BOOL FALSE +# # XY42 BOOL FALSE +# XY42 STR 0000000000000000000100000001000000000000000000000000000000000000 +# # XY43 STR 0000000000000000000000010000000100000000000010000100000100000100 +# XY43 STR 0000000000000000000000000000000000000000000010000000000000000000 +# XY44 BOOL FALSE +# XY45 BOOL FALSE +# XY46 BOOL FALSE +# XY47 BOOL FALSE +# XY50 BOOL FALSE +# XY51 BOOL FALSE +# XY52 BOOL FALSE +# XY53 BOOL FALSE +# XY54 BOOL FALSE +# XY55 BOOL FALSE +# XY56 BOOL FALSE +# XY57 BOOL FALSE +# XY60 BOOL FALSE +# XY61 BOOL FALSE +# # XY62 STR 0000000000000000000000000000000000000000000000000000000000000010 +# XY62 BOOL FALSE +# XY63 BOOL FALSE +# XY64 BOOL FALSE +# XY65 BOOL FALSE +# XY66 BOOL FALSE +# XY67 BOOL FALSE +# XY71 BOOL FALSE +# # XY72 STR 0000000000000000000000000000000000000000000000000000000000000001 +# XY72 BOOL FALSE +# # XY73 STR 0001100000000000000000000000001000000000000000000000000000000000 +# XY73 BOOL FALSE +# XY74 BOOL FALSE +# XY75 BOOL FALSE +# XY76 BOOL FALSE +# END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/ppImage.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/ppImage.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/ppImage.config (revision 33415) @@ -166,5 +166,36 @@ MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails PATTERN.ROW BOOL TRUE # Row pattern correction - PATTERN.CELL BOOL TRUE # Cell pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.CONTINUITY BOOL TRUE # Cell pattern correction using edge continuity + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? +END + +# Chip processing for testing pattern correction +CHIP_PATTERNTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL TRUE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? @@ -1083,4 +1114,31 @@ END +PR_NOPATTERN METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL TRUE # apply non-linearity correction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + REMNANCE BOOL FALSE # Remnance masking + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + MASK.BUILD BOOL TRUE # Build internal mask? + FRINGE BOOL FALSE # Fringe subtraction + PATTERN.ROW BOOL FALSE # Row pattern correction + PATTERN.CELL BOOL FALSE # Cell pattern correction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL FALSE # Subtract background? + CROSSTALK.MEASURE BOOL TRUE # Subtract model background? +END # Standard chip processing without any background removal @@ -1102,6 +1160,6 @@ MASK.BUILD BOOL TRUE # Build internal mask? MASK.BURNTOOL BOOL TRUE # Mask potential burntool trails - PATTERN.ROW BOOL FALSE # Row pattern correction - PATTERN.CELL BOOL FALSE # Cell pattern correction + PATTERN.ROW BOOL TRUE # Row pattern correction + PATTERN.CELL BOOL TRUE # Cell pattern correction FRINGE BOOL TRUE # Fringe subtraction BIN1.FITS BOOL TRUE # Save 1st binned chip image? Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/ppStack.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/ppStack.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/ppStack.config (revision 33415) @@ -36,12 +36,12 @@ END -# for stages that want just unconvolved and deep stacks +# for just unconvolved and deep stacks QUICKSTACK_DEEP METADATA - CONVOLVE BOOL FALSE - COMBINE.REJ F32 3.5 # Rejection threshold in combination (sigma). tune? - COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean. tune? + CONVOLVE BOOL FALSE + COMBINE.REJ F32 3.5 # Rejection threshold in combination (sigma). tune? + COMBINE.DISCARD F32 0.2 # Discard fraction for Olympic weighted mean. tune? OUTPUT.NOCOMP BOOL TRUE OUTPUT.LOGFLUX BOOL FALSE - PSF.INPUT.MAX F32 7.5 # since not convolved limit, relax? to tune. + PSF.INPUT.MAX F32 8.0 # since not convolved limit, relax? tune? PSF.INPUT.CLIP.NSIGMA F32 5.0 END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/psastro.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/psastro.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/psastro.config (revision 33415) @@ -344,4 +344,6 @@ PSASTRO.MATCH.RADIUS.N7 F32 5 PSASTRO.MATCH.FIT.NITER S32 5 + + PSASTRO.CATDIR STR STS.PP5.REFCAT END Index: branches/meh_branches/ppstack_test/ippconfig/gpc1/psphot.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/gpc1/psphot.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/gpc1/psphot.config (revision 33415) @@ -54,2 +54,6 @@ SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background END + +CHIP_CONSTANT_WEIGHTPHOT METADATA + CONSTANT_PHOTOMETRIC_WEIGHTS BOOL TRUE +END Index: branches/meh_branches/ppstack_test/ippconfig/lbc_red/filerules.mdc =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/lbc_red/filerules.mdc (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/lbc_red/filerules.mdc (revision 33415) @@ -136,4 +136,5 @@ PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE PSASTRO.OUT.ASTROM OUTPUT {OUTPUT}.ast ASTROM NONE FPA TRUE NONE Index: branches/meh_branches/ppstack_test/ippconfig/megacam/camera.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/megacam/camera.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/megacam/camera.config (revision 33415) @@ -154,4 +154,18 @@ PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.N} # Rule for generating photcode +BURNTOOL.STATE.GOOD S16 0 # Value for burntool_state with the most recent bt version. +BURNTOOL.STATE.GOOD.UPDATE S16 0 # for upddate processing accept earlier version + +FOV_REF F32 0 # Field of view of unvignetted region in FPA pixels. +FOV_MAX F32 0 +NPIX_REF S32 0 +NPIX_MAX S32 0 +NPIX_INTERCHIP S32 0 + +# This sets the maximum size for the fwhm_major. This is roughly 3", and 98.67% of all exposures in the database are less than this. +MAX_ALLOWED_FWHM F32 25 + # don't censor any masked pixels MASK.NO.CENSOR U32 4294967295 + +METADATA.COMPRESSION STR 7f # compression mode (nM): n = 1-9, M = f (filtered), h (Huffman), R (run-length) Index: branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-mef.mdc =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-mef.mdc (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-mef.mdc (revision 33415) @@ -42,4 +42,5 @@ PSASTRO.INPUT STR PSASTRO.INPUT.CMF PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF +#PSASTRO.OUTPUT.CMF STR PSASTRO.OUT.CMF DVOCORR.OUTPUT STR DVOCORR.MEF.OUTPUT DVOFLAT.OUTPUT STR DVOFLAT.MEF.OUTPUT @@ -243,4 +244,6 @@ PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE SPLIT PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE CHIP TRUE NONE +#PSASTRO.OUT.CMF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE NONE PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE @@ -308,4 +311,6 @@ PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG.FLIP OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE PPSTAMP.CHIP.MEF OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF Index: branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-simple.mdc =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-simple.mdc (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-simple.mdc (revision 33415) @@ -29,4 +29,11 @@ PPIMAGE.SHUTTER INPUT @DETDB CHIP IMAGE PPIMAGE.LINEARITY INPUT @DETDB CHIP LINEARITY + +## Files used by psvideophot +PSVIDEOPHOT.INPUT INPUT @FILES CHIP IMAGE +PSVIDEOPHOT.INPUT.MASK INPUT @FILES CHIP MASK +PSVIDEOPHOT.INPUT.VARIANCE INPUT @FILES CHIP VARIANCE +PSVIDEOPHOT.INPUT.PSF INPUT @FILES CHIP PSF +PSVIDEOPHOT.INPUT.SRC INPUT @FILES CHIP CMF ## Files used by ppMerge @@ -193,4 +200,8 @@ PSPHOT.PSF.RAW.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE PSPHOT.PSF.SKY.SAVE OUTPUT {OUTPUT}.psf PSF NONE FPA TRUE NONE + +#psvideophot +PSVIDEOPHOT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.vid OBJ NONE CHIP TRUE NONE +PSVIDEOPHOT.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.vid OBJ NONE CHIP TRUE NONE # used by staticsky with single input @@ -280,4 +291,6 @@ PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG.FLIP OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE NONE Index: branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-split.mdc =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-split.mdc (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/filerules-split.mdc (revision 33415) @@ -13,5 +13,5 @@ SOURCE.PLOT.APRESID STR SOURCE.PLOT.RAW.APRESID -# PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF +PSASTRO.OUTPUT STR PSASTRO.OUT.CMF.MEF # PSASTRO.OUTPUT.SPL STR PSASTRO.OUT.CMF.SPL # PSASTRO.OUTPUT.MEF STR PSASTRO.OUT.CMF.MEF @@ -220,6 +220,5 @@ #psvideophot -PSVIDEOPHOT.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.vid OBJ NONE CHIP TRUE NONE -PSVIDEOPHOT.CHIP OUTPUT {OUTPUT}.{CHIP.NAME}.vid OBJ NONE CHIP TRUE NONE +PSVIDEOPHOT.OUTPUT OUTPUT {OUTPUT}.vpt OBJ NONE CELL TRUE NONE # used by staticsky with single input @@ -246,8 +245,8 @@ # PSASTRO.OUTPUT.CMP OUTPUT {OUTPUT}.{CHIP.NAME}.smp CMP NONE CHIP TRUE NONE # PSASTRO.OUT.CMF.SPL OUTPUT {OUTPUT}.{CHIP.NAME}.smf CMF NONE CHIP TRUE NONE -# PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF PSASTRO.OUT.MODEL OUTPUT {OUTPUT}.asm ASTROM.MODEL NONE FPA TRUE NONE PSASTRO.OUT.REFSTARS OUTPUT {OUTPUT}.aref.fits ASTROM.REFSTARS NONE FPA TRUE NONE -PSASTRO.OUTPUT OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUT.CMF.MEF OUTPUT {OUTPUT}.smf CMF NONE FPA TRUE MEF +PSASTRO.OUTPUT.CMF OUTPUT {OUTPUT}.cmf CMF NONE FPA TRUE NONE PSASTRO.STATS OUTPUT {OUTPUT}.stats STATS NONE FPA TRUE NONE PSASTRO.CONFIG OUTPUT {OUTPUT}.psastro.mdc TEXT NONE FPA TRUE NONE @@ -323,7 +322,8 @@ PPSTAMP.OUTPUT.MASK OUTPUT {OUTPUT}.mk.fits MASK COMP_MASK FPA TRUE NONE PPSTAMP.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits VARIANCE COMP_WT FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE +PPSTAMP.OUTPUT.JPEG.FLIP OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE # mosaic image used for making stamps of raw images. Not saved. PPSTAMP.CHIP OUTPUT {OUTPUT}.ch.fits IMAGE NONE CHIP FALSE MEF -PPSTAMP.OUTPUT.JPEG1 OUTPUT {OUTPUT}.jpg JPEG NONE FPA TRUE NONE PPSIM.OUTPUT OUTPUT {OUTPUT}.{CHIP.NAME}.fits IMAGE NONE CHIP TRUE SPLIT Index: branches/meh_branches/ppstack_test/ippconfig/recipes/jpeg.mdc =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/jpeg.mdc (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/jpeg.mdc (revision 33415) @@ -254,2 +254,17 @@ END +PPSTAMP.OUTPUT.JPEG METADATA + COLORMAP STR greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +10.0 + OPTIONS STR -SB,+X,+Y +END + +PPSTAMP.OUTPUT.JPEG.FLIP METADATA + COLORMAP STR greyscale + SCALE.MODE STR RANGE + SCALE.MIN F32 -5.0 + SCALE.MAX F32 +10.0 + OPTIONS STR -SB,-X,+Y +END Index: branches/meh_branches/ppstack_test/ippconfig/recipes/nightly_science.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/nightly_science.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/nightly_science.config (revision 33415) @@ -2,15 +2,15 @@ CLEAN_MODES METADATA MODE STR CHIP - COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked + COMMAND STR chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 21 END CLEAN_MODES METADATA MODE STR WARP - COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked + COMMAND STR warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME U16 7 END CLEAN_MODES METADATA MODE STR DIFF - COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ -destreaked + COMMAND STR difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label @LABEL@ -data_group @DATA_GROUP@ RETENTION_TIME S16 21 END @@ -93,5 +93,5 @@ NAME STR MD02 DISTRIBUTION STR MD02 - TESS STR MD02.V2 + TESS STR MD02.V3 OBSMODE STR MD OBJECT STR MD02% @@ -104,5 +104,5 @@ NAME STR MD03 DISTRIBUTION STR MD03 - TESS STR MD03.V2 + TESS STR MD03.V3 OBSMODE STR MD OBJECT STR MD03% @@ -115,5 +115,5 @@ NAME STR MD04 DISTRIBUTION STR MD04 - TESS STR MD04.V2 + TESS STR MD04.V3 OBSMODE STR MD OBJECT STR MD04% @@ -126,5 +126,5 @@ NAME STR MD05 DISTRIBUTION STR MD05 - TESS STR MD05.V2 + TESS STR MD05.V3 OBSMODE STR MD OBJECT STR MD05% @@ -137,5 +137,5 @@ NAME STR MD06 DISTRIBUTION STR MD06 - TESS STR MD06 + TESS STR MD06.V3 OBSMODE STR MD OBJECT STR MD06% @@ -148,5 +148,5 @@ NAME STR MD07 DISTRIBUTION STR MD07 - TESS STR MD07 + TESS STR MD07.V3 OBSMODE STR MD OBJECT STR MD07% @@ -159,5 +159,5 @@ NAME STR MD08 DISTRIBUTION STR MD08 - TESS STR MD08.V2 + TESS STR MD08.V3 OBSMODE STR MD OBJECT STR MD08% @@ -170,5 +170,5 @@ NAME STR MD09 DISTRIBUTION STR MD09 - TESS STR MD09.V2 + TESS STR MD09.V3 OBSMODE STR MD OBJECT STR MD09% @@ -192,5 +192,5 @@ NAME STR MD11 DISTRIBUTION STR MD11 - TESS STR MD11.V2 + TESS STR MD11.V3 OBSMODE STR MD OBJECT STR MD11% @@ -210,7 +210,7 @@ REDUCTION STR STS_DATASET # bump the retention times for the time being - CHIP S16 60 - WARP S16 60 - DIFF S16 60 + CHIP S16 7 + WARP S16 7 + DIFF S16 7 END TARGETS METADATA @@ -219,6 +219,8 @@ TESS STR RINGS.V3 OBSMODE STR SS - STACKABLE BOOL FALSE + STACKABLE BOOL TRUE DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG # WARP S16 60 END @@ -228,6 +230,8 @@ TESS STR RINGS.V3 OBSMODE STR OSS - STACKABLE BOOL FALSE - DIFFABLE BOOL TRUE + STACKABLE BOOL TRUE + DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG END TARGETS METADATA @@ -236,6 +240,8 @@ TESS STR RINGS.V3 OBSMODE STR ESS - STACKABLE BOOL FALSE - DIFFABLE BOOL TRUE + STACKABLE BOOL TRUE + DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG END TARGETS METADATA @@ -244,6 +250,8 @@ TESS STR RINGS.V3 OBSMODE STR MSS - STACKABLE BOOL FALSE - DIFFABLE BOOL TRUE + STACKABLE BOOL TRUE + DIFFABLE BOOL TRUE + REDUCTION STR SWEETSPOT +# REDUCTION STR SSTF_4SIG END TARGETS METADATA @@ -262,4 +270,6 @@ STACKABLE BOOL FALSE DIFFABLE BOOL FALSE + CHIP S16 800 + WARP S16 800 END TARGETS METADATA Index: branches/meh_branches/ppstack_test/ippconfig/recipes/ppImage.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/ppImage.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/ppImage.config (revision 33415) @@ -27,4 +27,7 @@ PATTERN.SUBSET METADATA END +PATTERN.CONTINUITY BOOL FALSE +PATTERN.CONTINUITY.WIDTH S32 50 +PATTERN.CONTINUITY.SUBSET BOOL TRUE FRINGE BOOL FALSE # Fringe subtraction PHOTOM BOOL FALSE # Source identification and photometry @@ -177,4 +180,31 @@ BACKGROUND BOOL TRUE # Subtract background? MASK.STATS BOOL TRUE # Calculate Mask statistics. +END + +# Standard chip processing +CHIP_PATTERNTEST METADATA + BASE.FITS BOOL FALSE # Save base detrended image? + BASE.MASK.FITS BOOL FALSE # Save base detrended image? + BASE.VARIANCE.FITS BOOL FALSE # Save base detrended image? + CHIP.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.MASK.FITS BOOL TRUE # Save chip-mosaic-ed image? + CHIP.VARIANCE.FITS BOOL TRUE # Save chip-mosaic-ed image? + OVERSCAN BOOL TRUE # Overscan subtraction + NONLIN BOOL FALSE # Non-linearity correction; not implemented + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL TRUE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction + BIN1.FITS BOOL TRUE # Save 1st binned chip image? + BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + PHOTOM BOOL TRUE # Source identification and photometry + ASTROM.CHIP BOOL FALSE # Astrometry per chip? + ASTROM.MOSAIC BOOL FALSE # Astrometry for mosaic? + BACKGROUND BOOL TRUE # Subtract background? + MASK.STATS BOOL TRUE # Calculate Mask statistics. + PATTERN.CELL BOOL FALSE + PATTERN.ROW BOOL FALSE END Index: branches/meh_branches/ppstack_test/ippconfig/recipes/ppSim.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/ppSim.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/ppSim.config (revision 33415) @@ -46,11 +46,11 @@ FORCED.PHOTCODE STR r # source of the forced photometry positions -STARS.REAL BOOL TRUE # Add stars from a catalogue? -MATCH.DENSITY BOOL TRUE # significance of faintest sources +STARS.REAL BOOL TRUE # Add stars from a catalogue? +MATCH.DENSITY BOOL FALSE # significance of faintest sources STARS.FAKE BOOL TRUE # Add fake stars, randomly distributed? -STARS.LUM F32 0.35 # Stellar luminosity function slope (magnitude slope) -STARS.MAG F32 15.5 # Brightest magnitude for fake stars (choose based on real catalog limit) -STARS.DENSITY F32 1.0 # Stellar density (per square degree) at the brightest magnitude +STARS.LUM F32 0.40 # Stellar luminosity function slope (magnitude slope) +STARS.MAG F32 12.5 # Brightest magnitude for fake stars (choose based on real catalog limit) +STARS.DENSITY F32 100.0 # Stellar density (per square degree) at the brightest magnitude STARS.SIGMA.LIM F32 1.0 # significance of faintest sources @@ -69,5 +69,5 @@ GALAXY.FAKE BOOL FALSE # Generate fake galaxies? -GALAXY.LUM F32 -0.8 # Stellar luminosity function slope +GALAXY.LUM F32 0.8 # Stellar luminosity function slope GALAXY.MAG F32 15.0 # Brightest magnitude for fake stars GALAXY.DENSITY F32 5.0 # Galaxy density (per square degree) at the brightest magnitude @@ -83,11 +83,14 @@ GALAXY.THETA.MIN F32 0.0 # Minimum position angle for galaxy models -GALAXY.THETA.MAX F32 3.14 # Maximum position angle for galaxy models +GALAXY.THETA.MAX F32 180.0 # Maximum position angle for galaxy models GALAXY.GRID BOOL FALSE # Generate a (regular) grid of galaxy models? -GALAXY.GRID.DX S32 100 # Spacing between galaxies in grid in x -GALAXY.GRID.DY S32 200 # Spacing between galaxies in grid in y +GALAXY.GRID.DX S32 100 # Spacing between galaxies in grid in x +GALAXY.GRID.DY S32 200 # Spacing between galaxies in grid in y GALAXY.GRID.RANDOM BOOL FALSE # randomize shape parameters in grid (or sequence?) -GALAXY.GRID.PEAK F32 100.0 # peak flux of the grid galaxies +GALAXY.GRID.PEAK F32 100.0 # peak flux of the grid galaxies (deprecated) +GALAXY.GRID.MAG F32 17.0 # integrated magnitude of the grid galaxies +GALAXY.GRID.XOFF F32 0.0 # galaxy centroid offsets +GALAXY.GRID.YOFF F32 0.0 # galaxy centroid offsets GALAXY.MODEL STR PS_MODEL_GAUSS Index: branches/meh_branches/ppstack_test/ippconfig/recipes/ppStack.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/ppStack.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/ppStack.config (revision 33415) @@ -19,5 +19,5 @@ POOR.FRACTION F32 0.25 # Maximum fraction of bad variance for poor pixels THRESHOLD.MASK F32 0.5 # Threshold for mask deconvolution (0..1) -DECONV.LIMIT F32 2.0 # Deconvolution fraction for rejecting entire image +DECONV.LIMIT F32 3.0 # Deconvolution fraction for rejecting entire image IMAGE.REJ F32 0.1 # Rejected pixel fraction threshold for rejecting entire image MATCH.REJ F32 3.0 # Rejection threshold for chi^2 values from matching Index: branches/meh_branches/ppstack_test/ippconfig/recipes/ppStatsFromMetadata.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/ppStatsFromMetadata.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/ppStatsFromMetadata.config (revision 33415) @@ -536,2 +536,24 @@ # ENTRY VAL REPLICATE_OUTPUTS F32 CONSTANT -dtime_replicate # time to replicate the outputs END + +STATICSKY_CAL METADATA + ENTRY MULTI + TYPE VAL KEYWORD TYPE STATISTIC FLAG + ENTRY VAL CERROR F32 CONSTANT -sigma_ra + ENTRY VAL CERROR F32 CONSTANT -sigma_dec + ENTRY VAL DT_ASTR F32 CONSTANT -dtime_astrom + ENTRY VAL NASTRO S32 CONSTANT -n_astrom + + ENTRY VAL ZPT_OBS F32 CONSTANT -zpt_obs + ENTRY VAL ZPT_ERR F32 CONSTANT -zpt_stdev + + ENTRY VAL QUALITY S32 CONSTANT -quality # Bad quality flag + + # Revision values + ENTRY VAL PSLIB_V STR CONSTANT -ver_pslib + ENTRY VAL MODULE_V STR CONSTANT -ver_psmodules + ENTRY VAL PHOT_V STR CONSTANT -ver_psphot + ENTRY VAL ASTRO_V STR CONSTANT -ver_psastro + ENTRY VAL STATS_V STR CONSTANT -ver_ppstats +# ENTRY VAL IMAGE_V STR CONSTANT -ver_ppimage +END Index: branches/meh_branches/ppstack_test/ippconfig/recipes/ppSub.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/ppSub.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/ppSub.config (revision 33415) @@ -219,4 +219,13 @@ END +SSTF_DIFF METADATA + DUAL BOOL FALSE # Dual convolution? + INVERSE BOOL FALSE # Generate inverse subtraction? + PHOTOMETRY BOOL TRUE # Perform photometry? + NOCONVOLVE BOOL TRUE + CONVOLVE.TARGET STR SINGLE2 # convolution direction +END + + # Gene's choice: multiplying by 1.75 #@ISIS.WIDTHS F32 1.8 3.3 5.8 10.2 17.9 # Gaussian FWHMs for ISIS kernels Index: branches/meh_branches/ppstack_test/ippconfig/recipes/psastro.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/psastro.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/psastro.config (revision 33415) @@ -258,2 +258,9 @@ END +STATICSKY_CAL METADATA + REFSTAR_MASK BOOL FALSE + PSASTRO.SAVE.REFMATCH BOOL FALSE + PSASTRO.FIX.CHIPS BOOL FALSE + PSASTRO.USE.MODEL BOOL FALSE + PSASTRO.MOSAIC.MODE BOOL FALSE +END Index: branches/meh_branches/ppstack_test/ippconfig/recipes/psphot.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/psphot.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/psphot.config (revision 33415) @@ -199,7 +199,7 @@ #PGAUSS_PSF EXTENDED_SOURCE_MODEL PS_MODEL_PGAUSS 20.0 TRUE #QGAUSS_PSF EXTENDED_SOURCE_MODEL PS_MODEL_QGAUSS 20.0 TRUE - EXP_PCM EXTENDED_SOURCE_MODEL PS_MODEL_EXP 20.0 TRUE - DEV_PCM EXTENDED_SOURCE_MODEL PS_MODEL_DEV 20.0 TRUE - SER_PCM EXTENDED_SOURCE_MODEL PS_MODEL_SERSIC 20.0 TRUE + EXP_PCM EXTENDED_SOURCE_MODEL PS_MODEL_EXP 10.0 TRUE + DEV_PCM EXTENDED_SOURCE_MODEL PS_MODEL_DEV 10.0 TRUE + SER_PCM EXTENDED_SOURCE_MODEL PS_MODEL_SERSIC 10.0 TRUE END @@ -216,7 +216,7 @@ # @RADIAL.ANNULAR.BINS.UPPER F32 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 345.0 539.1 841.0 1305.0 # -# PS1 values? -@RADIAL.ANNULAR.BINS.LOWER F32 0.0 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 # perl parser fails without a comment ticket #1476 -@RADIAL.ANNULAR.BINS.UPPER F32 1.2 3.4 5.2 8.8 15.0 23.2 37.2 57.1 91.0 141.0 221.1 # perl parser fails without a comment ticket #1476 +# PS1 values? (revised fpr PS1 @0.25 arcsec pixel scales) +@RADIAL.ANNULAR.BINS.LOWER F32 0.0 0.96 2.72 4.16 7.04 12.0 18.56 29.76 45.68 72.80 112.80 # perl parser fails without a comment ticket #1476 still? +@RADIAL.ANNULAR.BINS.UPPER F32 0.96 2.72 4.16 7.04 12.0 18.56 29.76 45.68 72.80 112.80 176.88 # perl parser fails without a comment ticket #1476 still? # Extended source fit parameters @@ -329,9 +329,9 @@ # stack photometry parameters -PSPHOT.STACK.MATCH.RADIUS F32 1.0 # join detections from different images with this correlation radius (pixels) +PSPHOT.STACK.MATCH.RADIUS F32 5.0 # join detections from different images with this correlation radius (pixels) PSPHOT.STACK.MATCH.PSF BOOL T # convolved input images to common target PSF PSPHOT.STACK.MATCH.PSF.SOURCE STR AUTO # which inputs to convolve? (RAW, CNV, AUTO) PSPHOT.STACK.TARGET.PSF.AUTO BOOL F # automatically determine target PSF size? -@PSPHOT.STACK.TARGET.PSF.FWHM F32 6.5 9.0 # FWHM of target PSF (if NOT AUTO sized; pixels) +@PSPHOT.STACK.TARGET.PSF.FWHM F32 6.0 8.0 # FWHM of target PSF (if NOT AUTO sized; pixels) PSPHOT.STACK.USE.RAW BOOL T @@ -343,5 +343,5 @@ EXTENDED_SOURCE_FITS BOOL TRUE # perform any of the aperture-like measurements? EXTENDED_SOURCE_ANALYSIS BOOL TRUE # perform any of the aperture-like measurements? - EXTENDED_SOURCE_SN_LIM F32 20.0 + EXTENDED_SOURCE_SN_LIM F32 10.0 EXTENDED_SOURCE_PETROSIAN BOOL TRUE EXTENDED_SOURCE_ANNULI BOOL TRUE @@ -349,32 +349,36 @@ PSPHOT.STACK.TARGET.PSF.AUTO BOOL F # automatically determine target PSF size? PSPHOT.STACK.USE.RAW BOOL F # perform photometry & morphology analysis on the raw image - @PSPHOT.STACK.TARGET.PSF.FWHM F32 6.5 8.0 # FWHM of target PSF (if NOT AUTO sized; pixels) + @PSPHOT.STACK.TARGET.PSF.FWHM F32 6.0 8.0 # FWHM of target PSF (if NOT AUTO sized; pixels) RADIAL_APERTURES BOOL T # calculate flux in circular radial apertures? RADIAL_APERTURES_SN_LIM F32 0.0 # S/N limit for radial aperture calculation OUTPUT.FORMAT STR PS1_SV1 + PETROSIAN_FOR_STARS BOOL FALSE + + CONSTANT_PHOTOMETRIC_WEIGHTS BOOL TRUE + POISSON.ERRORS.PHOT.LMM BOOL TRUE EXT_FIT_MAX_RADIUS F32 50.0 - PSF_MODEL STR PS_MODEL_QGAUSS - EXT_MODEL STR PS_MODEL_SERSIC + PSF_MODEL STR PS_MODEL_GAUSS + EXT_MODEL STR PS_MODEL_QGAUSS END # Extended source fit parameters STACKPHOT_TEST METADATA - EXTENDED_SOURCE_FITS BOOL TRUE # perform any of the aperture-like measurements? - EXTENDED_SOURCE_ANALYSIS BOOL TRUE # perform any of the aperture-like measurements? - EXTENDED_SOURCE_SN_LIM F32 20.0 - EXTENDED_SOURCE_PETROSIAN BOOL TRUE - EXTENDED_SOURCE_ANNULI BOOL TRUE + EXTENDED_SOURCE_FITS BOOL F # perform any of the aperture-like measurements? + EXTENDED_SOURCE_ANALYSIS BOOL F # perform any of the aperture-like measurements? + EXTENDED_SOURCE_SN_LIM F32 10.0 + EXTENDED_SOURCE_PETROSIAN BOOL F + EXTENDED_SOURCE_ANNULI BOOL F PSPHOT.STACK.MATCH.PSF.SOURCE STR AUTO # which inputs to convolve? (RAW, CNV, AUTO) PSPHOT.STACK.TARGET.PSF.AUTO BOOL F # automatically determine target PSF size? PSPHOT.STACK.USE.RAW BOOL F # perform photometry & morphology analysis on the raw image - @PSPHOT.STACK.TARGET.PSF.FWHM F32 8.0 10.0 # FWHM of target PSF (if NOT AUTO sized; pixels) - RADIAL_APERTURES BOOL T # calculate flux in circular radial apertures? + @PSPHOT.STACK.TARGET.PSF.FWHM F32 6.0 8.0 # FWHM of target PSF (if NOT AUTO sized; pixels) + RADIAL_APERTURES BOOL F # calculate flux in circular radial apertures? RADIAL_APERTURES_SN_LIM F32 0.0 # S/N limit for radial aperture calculation OUTPUT.FORMAT STR PS1_SV1 EXT_FIT_MAX_RADIUS F32 50.0 - PSF_MODEL STR PS_MODEL_QGAUSS - EXT_MODEL STR PS_MODEL_SERSIC + PSF_MODEL STR PS_MODEL_GAUSS + EXT_MODEL STR PS_MODEL_QGAUSS END @@ -481,4 +485,7 @@ END +CHIP_CONSTANT_WEIGHTPHOT METADATA +END + # Recipe overrides for PSPHOT in sdssmosaic PSPHOT_NOAP_PSF METADATA @@ -500,4 +507,11 @@ OUTPUT.FORMAT STR PS1_DV2 # Format of output file END + +SSTF_DIFF METADATA + PEAKS_NSIGMA_LIMIT_2 F32 4.0 # peak significance threshold + DIFF_STATS BOOL TRUE # Calculate diff-specific stats + OUTPUT.FORMAT STR PS1_DV2 # Format of output file +END + # Recipe overrides for PR reduction @@ -513,2 +527,3 @@ SKY_STAT STR FITTED_MEAN_V4 # statistic used to measure background END + Index: branches/meh_branches/ppstack_test/ippconfig/recipes/reductionClasses.mdc =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/recipes/reductionClasses.mdc (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/recipes/reductionClasses.mdc (revision 33415) @@ -164,5 +164,8 @@ BACKGROUND_PSWARP STR BACKGROUND PSVIDEOPHOT STR PSVIDEOPHOT -END + STATICSKY_CALIBRATION STR STATICSKY_CAL +END + + # basic science analysis @@ -216,4 +219,52 @@ END +# Sweetspot reduction class +SWEETSPOT METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR QUICKSTACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + + +CONSTANT_WEIGHTPHOT METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP_CONSTANT_WEIGHTPHOT + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND +END + + DEFAULT_DARKTEST METADATA CHIP_PPIMAGE STR CHIP_DARKTEST @@ -239,4 +290,5 @@ + DEFAULT_MASKTEST METADATA CHIP_PPIMAGE STR CHIP_MASKTEST @@ -260,4 +312,76 @@ END +# SSTF tests +SSTF_NOCONV METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR SSTF_DIFF + DIFF_PSPHOT STR DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + +SSTF_4SIG METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + +SSTF_NOCONV_4SIG METADATA + CHIP_PPIMAGE STR CHIP + CHIP_PSPHOT STR CHIP + WARP_PSWARP STR WARP + STACK_PPSTACK STR STACK + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK + DIFF_PPSUB STR SSTF_DIFF + DIFF_PSPHOT STR SSTF_DIFF + JPEG_BIN1 STR PPIMAGE_J1 + JPEG_BIN2 STR PPIMAGE_J2 + FAKEPHOT STR FAKEPHOT + ADDSTAR STR ADDSTAR + PSASTRO STR DEFAULT_RECIPE + STACKPHOT_PSPHOT STR STACKPHOT + STACKPHOT_PPSUB STR STACKPHOT + STACKPHOT_PPSTACK STR STACKPHOT + STACKPHOT_SINGLE_PSPHOT STR STACKPHOT_SINGLE + BACKGROUND_PPBACKGROUND STR BACKGROUND + BACKGROUND_PSWARP STR BACKGROUND + PSVIDEOPHOT STR PSVIDEOPHOT +END + + + # reduction class specifically for warpwarp diffs: WARPWARP METADATA @@ -337,9 +461,8 @@ # quick deep stacks (for unconvolved refstacks and full,deep stacks) DEEP_QUICKSTACK METADATA - STACK_PPSTACK STR QUICKSTACK_DEEP - STACK_PPSUB STR STACK - STACK_PSPHOT STR STACK -END - + STACK_PPSTACK STR QUICKSTACK_DEEP + STACK_PPSUB STR STACK + STACK_PSPHOT STR STACK +END TESTSTACK METADATA Index: branches/meh_branches/ppstack_test/ippconfig/ssp/Makefile.am =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/ssp/Makefile.am (revision 33415) +++ branches/meh_branches/ppstack_test/ippconfig/ssp/Makefile.am (revision 33415) @@ -0,0 +1,18 @@ + +installdir = $(datadir)/ippconfig/ssp + +install_files = \ + camera.config \ + format.config \ + ppImage.config \ + psastro.config \ + psphot.config + +install_DATA = $(install_files) + +install-data-hook: + chmod 0755 $(installdir) + +EXTRA_DIST = $(install_files) + +ACLOCAL_AMFLAGS = -I m4 Index: branches/meh_branches/ppstack_test/ippconfig/ssp/camera.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/ssp/camera.config (revision 33415) +++ branches/meh_branches/ppstack_test/ippconfig/ssp/camera.config (revision 33415) @@ -0,0 +1,68 @@ +# Camera configuration file for drop-dead boring camera, consisting of a single chip with single cell. + +# File formats that we know about +FORMATS METADATA + SSP STR ssp/format.config +END + +# Description of camera --- all the chips and the cells that comprise them +FPA METADATA + Chip STR Cell +END + +# move this elsewhere? we need a lookup table to go from filter ID to abstract name +FILTER.ID METADATA + B STR B + V STR V + R STR R + I STR I + z STR z +END + +# Table of strings to use for the class identifier (see ippdb) according to the file level. +CLASSID METADATA + FPA STR fpa +END + +DVO.CAMERADIR STR megacam # Camera directory for DVO + +# convert supplied FPA.OBSTYPE values to abstract exptype names +OBSTYPE.TABLE METADATA + bias STR BIAS + zero STR BIAS + dark STR DARK + flat STR SKYFLAT + skyflat STR SKYFLAT + domeflat STR DOMEFLAT + object STR OBJECT + science STR OBJECT +END + +# Recipe options +RECIPES METADATA + PPIMAGE STR ssp/ppImage.config # Default: all (normal) options on + PSPHOT STR ssp/psphot.config # psphot details +END + +PHOTCODE.RULE STR {DETECTOR}.{FILTER.ID}.{CHIP.NAME} # Rule for generating photcode + +# reduction classes (recipes which are grouped together) +REDUCTION STR recipes/reductionClasses.mdc + +FITSTYPES STR recipes/fitstypes.mdc + +FILERULES STR recipes/filerules-simple.mdc + +EXTNAME.RULES METADATA + CMF.HEAD STR hdr + CMF.DATA STR psf + CMF.XSRC STR xsrc + CMF.XFIT STR xfit + CMF.DETEFF STR deteff + + PSF.HEAD STR hdr + PSF.TABLE STR psf_model + PSF.RESID STR psf_resid +END + +MASK.FORCE BOOL TRUE # Force reliance on MASKS recipe if headers not present Index: branches/meh_branches/ppstack_test/ippconfig/ssp/format.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/ssp/format.config (revision 33415) +++ branches/meh_branches/ppstack_test/ippconfig/ssp/format.config (revision 33415) @@ -0,0 +1,80 @@ +# Drop-dead simple camera with single chip with single cell + +# How to identify this type +RULE METADATA + SIMPLE BOOL TRUE +### NAXIS S32 2 +END + +# How to read this data +FILE METADATA + PHU STR FPA # The FITS file represents an entire FPA + EXTENSIONS STR NONE # There are no extensions + FPA.OBS STR NAXIS # A PHU keyword for unique identifier within the hierarchy level +END + +# What's in the FITS file? +CONTENTS STR Chip:Cell:amplifier + +# Specify the cell data +CELLS METADATA + amplifier METADATA + CELL.TRIMSEC.SOURCE STR VALUE + #CELL.TRIMSEC STR [0:0,0:0] + + # alternatives for getting these from the headers + # CELL.TRIMSEC.SOURCE STR HEADER + CELL.BIASSEC.SOURCE STR VALUE + CELL.TRIMSEC STR DATASEC + CELL.BIASSEC STR BIASSEC + END +END + +# How to translate PS concepts into FITS headers +TRANSLATION METADATA + #FPA.TELESCOPE STR TELESCOP + FPA.INSTRUMENT STR INSTRUME + FPA.OBSTYPE STR OBSTYPE + FPA.AIRMASS STR AIRMASS + FPA.FILTER STR FILTER + FPA.POSANGLE STR POSANGLE + FPA.RA STR RA + FPA.DEC STR DEC + FPA.OBJECT STR OBJECT + FPA.TIME STR DATE-OBS # UTC-OBS # Date and time + FPA.EXPOSURE STR EXPTIME + CELL.EXPOSURE STR EXPTIME + CELL.DARKTIME STR DARKTIME + FPA.FOCUS STR FOCUS + FPA.DETECTOR STR DETECTOR + CELL.GAIN STR CCDGAIN +# CELL.TIME STR DATE-OBS TIME-OBS # Date and time +END + +# Default PS concepts that may be specified by value +DEFAULTS METADATA + FPA.TELESCOPE STR SSP + CELL.XBIN S32 1 + CELL.YBIN S32 1 + FPA.TIMESYS STR UTC + FPA.RADECSYS STR ICRS + #CELL.GAIN F32 1.0 + CELL.READNOISE F32 2.0 + CELL.BAD F32 0 + CELL.SATURATION F32 65535 +# CELL.BAD F32 -65535 + CELL.READDIR S32 1 + CELL.TIMESYS STR UTC + CELL.XPARITY S32 1 + CELL.YPARITY S32 1 + CHIP.XPARITY S32 1 + CHIP.YPARITY S32 1 + CELL.X0 S32 0 + CELL.Y0 S32 0 +END + +FORMATS METADATA + FPA.RA STR HOURS + FPA.DEC STR DEGREES + FPA.TIME STR SEPARATE,YEAR.FIRST +END Index: branches/meh_branches/ppstack_test/ippconfig/ssp/ppImage.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/ssp/ppImage.config (revision 33415) +++ branches/meh_branches/ppstack_test/ippconfig/ssp/ppImage.config (revision 33415) @@ -0,0 +1,120 @@ +### ppImage recipe configuration file for drop-dead simple camera + +# List of tasks to perform +SHUTTER BOOL FALSE # Shutter correction +DARK BOOL TRUE # Dark correction + +BASE.FITS BOOL TRUE # Save base detrended image? +BIN1.FITS BOOL TRUE # Save 1st binned chip image? +BIN2.FITS BOOL TRUE # Save 2nd binned chip image? + +# binned output image options +BIN1.XBIN S32 8 +BIN1.YBIN S32 8 +BIN2.XBIN S32 64 +BIN2.YBIN S32 64 + +# Overscan subtraction +OVERSCAN.SINGLE BOOL FALSE # Reduce overscan to a single value? +OVERSCAN.FIT STR POLYNOMIAL # NONE | POLYNOMIAL | SPLINE +OVERSCAN.ORDER S32 3 # Order of polynomial fit +OVERSCAN.STAT STR MEAN # MEAN | MEDIAN + +# How to select the appropriate detrend image +DETREND.CONSTRAINTS METADATA + BIAS METADATA + END + MASK METADATA + END + DARK METADATA + END + FLAT METADATA + FILTER STR FPA.FILTER + END + FRINGE METADATA + FILTER STR FPA.FILTER + END + SHUTTER METADATA + END +END + +######################################################################################## +# Need the following in order to turn off overscan and shutter for the site-level recipe +######################################################################################## + +# Overscan subtraction only +PPIMAGE_O METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Dark subtraction only +PPIMAGE_D METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Shutter correction only +PPIMAGE_S METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL FALSE # Bias subtraction + DARK BOOL FALSE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark +PPIMAGE_OBD METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark, shutter +PPIMAGE_OBDS METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL FALSE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark, shutter, flat-field +PPIMAGE_OBDSF METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL FALSE # Fringe subtraction +END + +# Overscan, bias, dark, shutter, flat-field, fringe +PPIMAGE_OBDSFR METADATA + OVERSCAN BOOL FALSE # Overscan subtraction + BIAS BOOL TRUE # Bias subtraction + DARK BOOL TRUE # Dark subtraction + SHUTTER BOOL FALSE # Shutter correction + FLAT BOOL TRUE # Flat-field normalisation + MASK BOOL FALSE # Mask bad pixels + FRINGE BOOL TRUE # Fringe subtraction +END Index: branches/meh_branches/ppstack_test/ippconfig/ssp/psastro.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/ssp/psastro.config (revision 33415) +++ branches/meh_branches/ppstack_test/ippconfig/ssp/psastro.config (revision 33415) @@ -0,0 +1,36 @@ + +# astrometry matching parameters + +# nominal plate scale (microns / pixel) +# PSASTRO.PLATE.SCALE F32 13.5 +PSASTRO.PLATE.SCALE F32 1.0 + +# pmAstromGridMatch: +PSASTRO.GRID.MIN.ANGLE F32 -0.0851 +PSASTRO.GRID.MAX.ANGLE F32 +0.0851 +PSASTRO.GRID.DEL.ANGLE F32 0.0170 + +# pmAstromGridAngle +# max grid offset in FP units (microns) +# use plate-scale to make this in pixels? +# PSASTRO.GRID.OFFSET F32 10000. +# PSASTRO.GRID.SCALE F32 500. +PSASTRO.GRID.OFFSET F32 1000. +PSASTRO.GRID.SCALE F32 50 + +# these tweak are in FP units (pixels, currently) +PSASTRO.TWEAK.SCALE F32 1 +PSASTRO.TWEAK.RANGE F32 75 +PSASTRO.TWEAK.SMOOTH F32 2 +PSASTRO.TWEAK.NSIGMA F32 3 + +# pmAstromRadiusMatch +# use plate-scale to make this in pixels? +PSASTRO.MATCH.RADIUS F32 8 + +# pmAstromMatchFit +PSASTRO.CHIP.NX S32 1 # fit order +PSASTRO.CHIP.NY S32 1 # fit order +PSASTRO.CHIP.NITER S32 3 # fit clipping iterations +PSASTRO.CHIP.NSIGMA F32 3 # fit clipping sigmas + Index: branches/meh_branches/ppstack_test/ippconfig/ssp/psphot.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/ssp/psphot.config (revision 33415) +++ branches/meh_branches/ppstack_test/ippconfig/ssp/psphot.config (revision 33415) @@ -0,0 +1,2 @@ + +SAVE.RESID BOOL TRUE Index: branches/meh_branches/ppstack_test/ippconfig/system.config =================================================================== --- branches/meh_branches/ppstack_test/ippconfig/system.config (revision 32161) +++ branches/meh_branches/ppstack_test/ippconfig/system.config (revision 33415) @@ -16,4 +16,5 @@ VYSOS5 STR vysos5/camera.config # VYSOS 5in VYSOS20 STR vysos20/camera.config # VYSOS 5in + SSP STR ssp/camera.config # Pan-STARRS spectroscopic sky probe SIMMOSAIC STR simmosaic/camera.config # Simulated mosaic, for testing SIMTEST STR simtest/camera.config # Simulation test camera Index: branches/meh_branches/ppstack_test/ippdor/IppSiteTest.py =================================================================== --- branches/meh_branches/ppstack_test/ippdor/IppSiteTest.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/IppSiteTest.py (revision 33415) @@ -0,0 +1,37 @@ +""" +This module contains the definitions relative to the installation and +the configuration of the IPP-Condor framework. +""" +class IppSite: + """ + This module contains the definitions relative to the installation + and the configuration of the IPP-Condor framework. + """ + def __init__(self): + """Not to be instantiated + """ + raise Exception('Not to be instantiated') + + SVN_SOURCE_URL = 'https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/trunk/ippdor/src' + TARGET_DIRECTORY = '/home/schastel/dev/ippdor/test/ipp' + TARGET_BINARY_DIRECTORY = '/home/schastel/dev/ippdor/test/bin' + + LOCAL_TMP_DIR = '/tmp/ippdor_processing' + NFS_TMP_DIR = '~/ippdor_processing' + IPP_BIN_PATH = '~/ipp/default.linux' + +class Gpc1: + """ + Information relative to the Gpc1 database + + NOT INTENDED TO BE USABLE + """ + def __init__(self): + """Not to be instantiated + """ + raise Exception('Not to be instantiated') + HOST = 'somehost.ipp.ifa.hawaii.edu' + NAME = 'gpc1' + USER = 'username' + PASSWORD = 'its_password' + Index: branches/meh_branches/ppstack_test/ippdor/README =================================================================== --- branches/meh_branches/ppstack_test/ippdor/README (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/README (revision 33415) @@ -0,0 +1,10 @@ +Installation: + ./deployment/deploy.py -h + +Documentation: + cd ./doc && make + +Code Quality Metrics (pylint 0.25 required): + ./quality_metrics.sh + + Index: branches/meh_branches/ppstack_test/ippdor/conf/pylintrc =================================================================== --- branches/meh_branches/ppstack_test/ippdor/conf/pylintrc (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/conf/pylintrc (revision 33415) @@ -0,0 +1,310 @@ +# lint Python modules using external checkers. +# +# This is the main checker controlling the other ones and the reports +# generation. It is itself both a raw checker and an astng checker in order +# to: +# * handle message activation / deactivation at the module level +# * handle some basic but necessary stats'data (number of classes, methods...) +# +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# Set the cache size for astng objects. +cache-size=500 + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable only checker(s) with the given id(s). This option conflicts with the +# disable-checker option +#enable-checker= + +# Enable all checker(s) except those with the given id(s). This option +# conflicts with the enable-checker option +#disable-checker= + +# Enable all messages in the listed categories (IRCWEF). +#enable-msg-cat= + +# Disable all messages in the listed categories (IRCWEF). +disable-msg-cat=I + +# Enable the message(s) with the given id(s). +#enable-msg= + +# Disable the message(s) with the given id(s). +disable-msg=W0704 + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Include message's id in output +include-ids=no + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (R0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (R0004). +comment=no + +# Enable the report(s) with the given id(s). +#enable-report= + +# Disable the report(s) with the given id(s). +#disable-report= + + +# checks for : +# * doc strings +# * modules / classes / functions / methods / arguments / variables name +# * number of arguments, local variables, branches, returns and statements in +# functions, methods +# * required module attributes +# * dangerous default values as arguments +# * redefinition of function / method / class +# * uses of the global statement +# +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + + +# try to find bugs in the code using type inference +# +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. +generated-members=REQUEST,acl_users,aq_parent + + +# checks for +# * unused variables / imports +# * undefined variables +# * redefinition of variable from builtins or from an outer scope +# * use of variable before assignment +# +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +# checks for : +# * methods without self as first argument +# * overridden methods signature +# * access only to existent members via self +# * attributes not defined in the __init__ method +# * supported interfaces implementation +# * unreachable code +# +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + + +# checks for +# * external modules dependencies +# * relative / wildcard imports +# * cyclic imports +# * uses of deprecated modules +# +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report R0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report R0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report R0402 must +# not be disabled) +int-import-graph= + + +# checks for sign of poor/misdesign: +# * number of methods, attributes, local variables... +# * size, complexity of functions, methods +# +[DESIGN] + +# Maximum number of arguments for function / method +max-args=8 + +# Maximum number of locals for function / method body +max-locals=25 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=15 + +# Maximum number of statements in function / method body +max-statements=150 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=0 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=30 + + +# checks for: +# * warning notes in the code like FIXME, XXX +# * PEP 263: source code with non ascii character but no encoding declaration +# +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +# checks for similarities and duplicated code. This computation may be +# memory / CPU intensive, so you should disable it if you experiments some +# problems. +# +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=10 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +# checks for : +# * unauthorized constructions +# * strict indentation +# * line length +# * use of <> instead of != +# +[FORMAT] + +# Maximum number of characters on a single line. +#max-line-length=80 +max-line-length=120 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' Index: branches/meh_branches/ppstack_test/ippdor/deployment/Deployment.py =================================================================== --- branches/meh_branches/ppstack_test/ippdor/deployment/Deployment.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/deployment/Deployment.py (revision 33415) @@ -0,0 +1,226 @@ +#!/usr/bin/env python + +"""Utility class for Condor IPP software deployent +""" + +import os +import subprocess +import tempfile +import shutil +import logging + +class Deployment: + """Utility class for Condor IPP software deployent + """ + logger = logging.getLogger('ippdor') + + CLASSES_FILENAMES = [ '__init__.py', + 'ChipManager.py', + 'CamManager.py', + 'FakeManager.py', + 'WarpManager.py', + 'StackManager.py', + 'Gpc1Manager.py', + 'helpers.py', + 'constants/__init__.py', + 'constants/Globals.py', + 'constants/Chip.py', + 'constants/Camera.py', + 'constants/Fake.py', + 'constants/Warp.py', + 'constants/Stack.py', + 'exceptions/__init__.py', + 'exceptions/IppException.py', + 'exceptions/UninstantiableIppException.py', + ] + EXECUTABLES_FILENAMES = [ 'generate_chip_jobs.py', + 'generate_cam_jobs.py', + 'generate_fake_jobs.py', + 'generate_warp_jobs.py', + 'generate_stack_jobs.py', + 'generate_exposure_dag.py', + 'ippdor_chip_to_warp.py', + # 'ippdor_chip_to_stack.py', + 'undo_dag_exposure.py', + 'send_email.py', + 'encapsulate_chip.sh', + 'encapsulate_camera.sh', + 'encapsulate_fake.sh', + 'encapsulate_warp.sh', + 'encapsulate_stack.sh', + ] + + def __init__(self, + svn_source_url, + target_directory, + target_binary_directory, + logger = None): + """ + Constructor: + + svn_source_url: SVN Source URL + target_directory: Target Python Directory + target_binary_directory: Target Executables Directory + logger: optional argument (if None: logging to logging.getLogger('ippdor')) + """ + self.svn_source_url = svn_source_url + self.target_directory = target_directory + self.target_binary_directory = target_binary_directory + self.temporary_directory = None # Set when necessary + if logger is not None: + Deployment.logger = logger + + def copy_to_targetdir(self, path_to_file, filename): + """ + Copy a file from the SVN temporary directory to the target + directory. + """ + Deployment.logger.info('...... Installing %s' % filename) + shutil.copyfile('%s/%s/%s' % (self.temporary_directory, + path_to_file, + filename), + '%s/%s' % (self.target_directory, filename)) + + def copy_to_bindir(self, path_to_file, filename): + """ + Copy a file from the SVN temporary directory to the target + binary directory. + """ + Deployment.logger.info('...... Installing %s' % filename) + shutil.copyfile('%s/%s/%s' % (self.temporary_directory, + path_to_file, + filename), + '%s/%s' % (self.target_binary_directory, filename)) + + def check(self): + """ + Check different elements + """ + Deployment.logger.info('Checking Condor-IPP Softwares Configuration') + # Configuration + from IppSite import IppSite, Gpc1 + Deployment.logger.info('Checking definition of IppSite.SVN_SOURCE_URL: %s' + % IppSite.SVN_SOURCE_URL) + Deployment.logger.info('Checking definition of IppSite.TARGET_DIRECTORY: %s' + % IppSite.TARGET_DIRECTORY) + Deployment.logger.info('Checking definition of IppSite.TARGET_BINARY_DIRECTORY: %s' + % IppSite.TARGET_BINARY_DIRECTORY) + Deployment.logger.info('Checking definition of IppSite.LOCAL_TMP_DIR: %s' + % IppSite.LOCAL_TMP_DIR) + Deployment.logger.info('Checking definition of IppSite.NFS_TMP_DIR: %s' + % IppSite.NFS_TMP_DIR) + Deployment.logger.info('Checking definition of IppSite.IPP_BIN_PATH: %s' + % IppSite.IPP_BIN_PATH) + Deployment.logger.info('Checking definition of Gpc1.HOST: %s' + % Gpc1.HOST) + Deployment.logger.info('Checking definition of Gpc1.NAME: %s' + % Gpc1.NAME) + Deployment.logger.info('Checking definition of Gpc1.USER: %s' + % Gpc1.USER) + Deployment.logger.info('Checking definition of Gpc1.PASSWORD: %s[Hidden]' + % Gpc1.PASSWORD[:0]) + # target_directory + Deployment.logger.info('... Checking write permission to %s' % self.target_directory) + extra = '' + if not os.path.exists(self.target_directory): + os.makedirs(self.target_directory) + extra = ' (directory was created)' + if not os.access(self.target_directory, os.W_OK): + raise IOError('Directory [%s] is not writable' % self.target_directory) + Deployment.logger.info('...... OK%s' % extra) + # target_binary_directory + Deployment.logger.info('... Checking write permission to %s' % self.target_binary_directory) + extra = '' + if not os.path.exists(self.target_binary_directory): + os.makedirs(self.target_binary_directory) + extra = ' (directory was created)' + if not os.access(self.target_binary_directory, os.W_OK): + raise IOError('Directory [%s] is not writable' % self.target_binary_directory) + Deployment.logger.info('...... OK%s' % extra) + # Check deployment software version? + # Anything else? + Deployment.logger.info('... Check complete') + + def svn_export(self): + """ + Export the software under configuration into the SVN temporary directory + """ + Deployment.logger.info('Updating Condor-IPP Softwares SVN temporary directory') + Deployment.logger.info('... Creating temporary directory:') + self.temporary_directory = tempfile.mkdtemp() + Deployment.logger.info('...... OK [%s]' % self.temporary_directory) + Deployment.logger.info('... Exporting Condor source code to temporary directory') + process = subprocess.Popen( ('svn export --force %s %s' % (self.svn_source_url, + self.temporary_directory)).split(' '), + stdout = open('/dev/null', 'w'), + stderr = open('/dev/null', 'w') ) + status = process.wait() + if status != 0: + Deployment.logger.error('Error: svn export failed') + raise Exception('svn export failed') + Deployment.logger.info('...... OK') + Deployment.logger.info('... SVN Update complete') + + def generate(self): + """ + Generate any file depending on the installation target + """ + Deployment.logger.info('Generating Condor-IPP Softwares') + Deployment.logger.debug('--- Nothing to generate yet from %s' % self.svn_source_url) + Deployment.logger.info('... Generation complete') + + def deploy(self): + """ + Deploy files on the target + """ + Deployment.logger.info('Deploying Condor-IPP Softwares') + Deployment.logger.info('... Installing Site definition class to [%s]' + % self.target_directory) + shutil.copyfile('.tmp_site/IppSite.py', + '%s/IppSite.py' % (self.target_directory)) + Deployment.logger.info('... Installing Python classes to [%s]' % self.target_directory) + for filename in Deployment.CLASSES_FILENAMES: + path_elements = filename.split('/') + if len(path_elements) > 1: + try: + os.makedirs('%s/%s' % (self.target_directory, + path_elements[0])) + except OSError, exc: + Deployment.logger.warn(exc) + Deployment.logger.info('Ignored') + for filename in Deployment.CLASSES_FILENAMES: + self.copy_to_targetdir('ipp', filename) + Deployment.logger.info('... Installing executables to [%s]' % self.target_binary_directory) + for filename in Deployment.EXECUTABLES_FILENAMES: + self.copy_to_bindir('executables', filename) + Deployment.logger.info('... Deployment complete') + + def clean(self, doit = False): + """ + Clean the installation, i.e. remove files and subdirectories + if doit is True (default is False). Otherwise, doesn't do + anything... + """ + Deployment.logger.info('Cleaning') + if not doit: + Deployment.logger.warn('!!! NO DELETION WILL BE PERFORMED SINCE doit IS False') + try: + Deployment.logger.info('... Deleting temporary directory [%s]' % self.temporary_directory) + if doit: + for root, dirs, files in os.walk(self.temporary_directory, topdown=False): + Deployment.logger.debug('...... Deleting [%s]' % root) + for name in files: + Deployment.logger.debug('...... Deleting [%s] (in files)' % name) + os.remove(os.path.join(root, name)) + for name in dirs: + Deployment.logger.debug('...... Deleting [%s] (in dirs)' % name) + os.rmdir(os.path.join(root, name)) + os.rmdir(self.temporary_directory) + except AttributeError: + Deployment.logger.info('... Nothing to be cleaned') + Deployment.logger.info('... Cleaning complete') + +if __name__ == '__main__': + print "Running doctest" + import doctest + doctest.testmod() Index: branches/meh_branches/ppstack_test/ippdor/deployment/__init__.py =================================================================== --- branches/meh_branches/ppstack_test/ippdor/deployment/__init__.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/deployment/__init__.py (revision 33415) @@ -0,0 +1,3 @@ +""" +This is the deployment module: a module to deploy Condor-IPP software. +""" Index: branches/meh_branches/ppstack_test/ippdor/deployment/deploy.py =================================================================== --- branches/meh_branches/ppstack_test/ippdor/deployment/deploy.py (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/deployment/deploy.py (revision 33415) @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +""" +Utility executable for IPP Condor deployment +""" +import sys +import os +import logging +#optparse Deprecated in python 2.7. Will be replaced by argparse +from optparse import OptionParser +from Deployment import Deployment + +if __name__ == '__main__': + if os.getenv('PYTHONPATH') is not None: + print 'Error: Unset PYTHONPATH environment variable to run this program' + sys.exit(1) + USAGE = """usage: %prog [options] + +Deploy the Condor-IPP softwares +""" + PARSER = OptionParser(usage = USAGE) + PARSER.set_defaults(verbose=True) + PARSER.add_option( '-v', action = 'store_true', + dest = 'verbose', + help = 'Be verbose', + default = False) + PARSER.add_option('-q', action='store_true', + dest = 'quiet', + help = 'Be quiet', + default = False) + (OPTIONS, ARGS) = PARSER.parse_args() # ... the values are stored in options + if len(ARGS) != 1: + PARSER.print_help() + sys.exit(1) + IPP_SITE_FILE = ARGS[0] + # Logging setup + LOGGER = logging.getLogger('ippdor') + FORMATTER = logging.Formatter('%(asctime)s | %(levelname)7s | %(message)s', + '%Y-%m-%dT%H:%M:%S') + STDOUT = logging.StreamHandler(sys.stdout) + STDOUT.setFormatter(FORMATTER) + LOGGER.addHandler(STDOUT) + if OPTIONS.verbose: + LOGGER.setLevel(logging.DEBUG) + LOGGER.debug('Logging level set to DEBUG') + elif OPTIONS.quiet: + LOGGER.setLevel(logging.ERROR) + LOGGER.info('Logging level set to ERROR') + else: + LOGGER.setLevel(logging.INFO) + # Import IppSite definition + LOGGER.info('Importing local site definitions') + try: + os.makedirs('.tmp_site') + except OSError: + LOGGER.warn('Existing .tmp_site directory') + import shutil + shutil.copy(IPP_SITE_FILE, '.tmp_site/IppSite.py') + INITFILE = open('.tmp_site/__init__.py', 'w') + INITFILE.close() + sys.path.append('.tmp_site') + from IppSite import IppSite + + # Deployment + try: + DEPLOYMENT = Deployment(IppSite.SVN_SOURCE_URL, + IppSite.TARGET_DIRECTORY, + IppSite.TARGET_BINARY_DIRECTORY) + DEPLOYMENT.check() + DEPLOYMENT.svn_export() + DEPLOYMENT.generate() + DEPLOYMENT.deploy() + DEPLOYMENT.clean(doit = True) + except Exception, e: + LOGGER.info('Exception caught') + DEPLOYMENT.clean(doit = False) + LOGGER.error(e) + import traceback + traceback.print_exc() + # Clean temporary site + LOGGER.info('Cleaning temporary site definitions') + os.remove('.tmp_site/IppSite.py') + try: + os.remove('.tmp_site/IppSite.pyc') + except OSError: + pass # Ignore if not compiled + os.remove('.tmp_site/__init__.py') + os.rmdir('.tmp_site') Index: branches/meh_branches/ppstack_test/ippdor/doc/Chip2Stack/stack.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Chip2Stack/stack.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Chip2Stack/stack.tex (revision 33415) @@ -0,0 +1,97 @@ +\documentclass[10pt]{article} +\usepackage[hmargin=0.5in,vmargin=0.5in]{geometry} + +\begin{document} + +\textbf{\Large FOR EACH EXPOSURE} + +\begin{description} +\item[Prerequisites] The exposures to be processed are + \textbf{labelled} externally in the chipRun table. +\item[A. Chip stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE A010] (Script) Generate the Condor job file containing the + \verb+chip_imfile.pl+ commands to be run on the cluster + \hfill{}\mbox{Equivalent to chip.imfile.load} + \item[A010] (Condor) Queue that Condor job file (exactly 60 + commands/jobs: one for each ota) \hfill{}\mbox{Equivalent to + chip.imfile.run} + \item[POST A010] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A020] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A020] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A030] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A030] (Script) Generate the Condor job file containing the + appropriate jobs for reverting \hfill{}\mbox{Equivalent to + chip.revert} + \item[A040] (Condor) Queue that Condor job file (various, possibly + nothing to do) + \item[POST A040] (Script) If there is failing otas, send e-mail to + operator and stop processing. Otherwise generate stage 050 jobs file. + \item[A050] (Script) Advance the exposure with + \verb+chiptool -advanceexp+ \hfill{}\mbox{Equivalent to + chip.advanceexp} + \end{description} +\item[B. Camera stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE B010] (Script) Generate the Condor job file containing the + \verb+camera_exp.pl+ command(s?) \hfill{}\mbox{Equivalent to + camera.exp.load} + \item[B010] (Condor) Queue that Condor job file + \hfill{}\mbox{Equivalent to camera.exp.run} + \item[PRE B020] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B020] (Condor) Queue that Condor job file + \item[PRE B030] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B030] (Condor) Queue that Condor job file + \item[PRE B040] (Script) Generate the Condor job file possibly + containing the revert command (possibly nothing to revert) + \hfill{}\mbox{Equivalent to camera.revert} + \item[B040] (Condor) Queue that Condor job file + \item[POST B040] (Script) If the camera entry could be reverted, send + e-mail to operator and stop processing + \end{description} +\item[C. Fake stage] Detail prerequisites \mbox{ } + \begin{description} + \item[PRE C010] (Script) Generate the Condor job file containing the + \verb+fake_imfile.pl+ command(s?) \hfill{}\mbox{Equivalent to + fake.imfile.load} + \item[C010] (Condor) Queue that Condor job file + \hfill{}\mbox{Equivalent to fake.imfile.run} + \item[C020] (Script (or Script+Condor)) Run the + \verb+faketool -advanceexp+ command(s?) (or: PRE C020: Generate + the Condor job file containing the \verb+faketool -advanceexp+ + command(s?); C020: Queue that Condor job file) + \hfill{}\mbox{Equivalent to fake.advanceexp} + \end{description} +\item[D. Warp stage] Detail prerequisites \hrulefill{}\textbf{BARF}\mbox{ } + \begin{description} + \item[TODO] \verb+warptool -tooverlap+ \hfill{}\mbox{Equivalent to warp.exp.load} + \item[TODO] \verb+warp_overlap.pl+ \hfill{}\mbox{Equivalent to warp.exp.run} + \item[TODO] \hfill{}\mbox{Equivalent to warp.revert.overlap} + \item[TODO] \verb+warptool -towarp+ \hfill{}\mbox{Equivalent to warp.skycell.load} + \item[TODO] \verb+warp_skycell.pl+ \hfill{}\mbox{Equivalent to warp.skycell.run} + \item[TODO] \verb++ \hfill{}\mbox{Equivalent to warp.revert.warped} + \item[TODO] \verb+ warptool -advancerun+ \hfill{}\mbox{Equivalent to warp.advancerun} + \item[Check if needed?] \verb++ \hfill{}\mbox{Equivalent to + warp.summary.load and/or warp.summary.run} + \end{description} +\item[E. Stack stage] Detail prerequisites? \mbox{ } + \begin{description} + \item[TODO] \verb+stacktool -tosum+ \hfill{}\mbox{Equivalent to stack.skycell.load} + \item[TODO] \verb+stack_skycell.pl + \hfill{}\mbox{Equivalent to stack.skycell.run} + \item[TODO] \verb++ \hfill{}\mbox{Equivalent to stack.revert} + \item[Check if needed?] \verb++ \hfill{}\mbox{Equivalent to + stack.summary.load and/or stack.summary.run} + \end{description} +\end{description} + +\end{document} Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/Makefile =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/Makefile (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/Makefile (revision 33415) @@ -0,0 +1,10 @@ +MAIN = ippdor + +all: + pdflatex $(MAIN).tex + makeindex $(MAIN) + pdflatex $(MAIN).tex + +clean: + /bin/rm -f $(MAIN).pdf + CleanTex.sh Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/administration.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/administration.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/administration.tex (revision 33415) @@ -0,0 +1,34 @@ + +\chapter{Administration} +\label{chap-adm} + +\section{The \texttt{ippdor} user} +\label{sec-adm-ippdor} + +\texttt{ippdor} is the dedicated user for any processing happening in +Condor. This user uses the \texttt{bash} shell. The ippdor has no +particular privilege on the system. + +The \texttt{bashrc} file contains some definitions and expansions: +\begin{itemize} +\item \texttt{PATH} adds \texttt{\$HOME/local/bin}; +\item \texttt{LD\_LIBRARY\_PATH} adds \texttt{\$HOME/local/lib}; +\item \texttt{PYTHONPATH} adds \texttt{\$HOME/local/share/python} and + \texttt{\$HOME/local/lib/python2.6/site-packages}; +\item The \texttt{PSCONFIG} value is fixed to \texttt{current}. See + section~\ref{sec-link-ippTODO} for details. The + \texttt{psconfig.bash} utility is called using that value. +\end{itemize} + +\texttt{cmake} was locally installed to allow Condor build and +install. It is now the only external software. It was built using the +\texttt{--prefix=\$HOME/local} option and can be found in the +\texttt{\$HOME/softwares} directory. + +\section{Deploying Condor on a new host} +\label{sec-adm-newnode} + +\section{Updating Condor} +\label{sec-adm-update} + +To be written when it's happening... Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/annex.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/annex.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/annex.tex (revision 33415) @@ -0,0 +1,103 @@ +\chapter{Annex} + +\section{Held Jobs} +\label{annex-heldjobs} + +\null\hfill{\parbox{5in}{\scriptsize This example is related to the + DAG shown in section~\ref{sec-oper-tasks}. The A.condor and + B.condor jobs are defined as "echo A" and "echo B" commands.}} + +If \texttt{condor\_submit\_dag} is executed from a local directory +(e.g. \textbf{/export/ippc01.0/ippdor/simple\_dag}), the jobs will be +indefinitely held by Condor. + +\begin{verbatim} +ippdor@ippc01 /export/ippc01.0/ippdor/simple_dag $ condor_submit_dag AB.dag +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71204. +----------------------------------------------------------------------- +\end{verbatim} + +This can be seen with the +\texttt{condor\_q} command: + +\begin{verbatim} +$ condor_q + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu + ID OWNER SUBMITTED RUN_TIME ST PRI SIZE CMD +71204.0 ippdor 11/25 11:41 0+00:01:27 R 0 12.2 condor_dagman +71205.0 ippdor 11/25 11:42 0+00:00:01 H 0 0.0 echo A + +2 jobs; 0 idle, 1 running, 1 held +\end{verbatim} + +``In-depth'' analysis can be performed with \texttt{condor\_q -analyze} command: + +\begin{verbatim} +$ condor_q -analyze 71205.0 + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu +--- +71205.000: Request is held. + +Hold reason: Error from slot7@ipp029.ifa.hawaii.edu: Failed to execute +'/bin/echo' with arguments A: Cannot access specified iwd +"/export/ippc01.0/ippdor/simple_dag" (errno=2: 'No such file or directory') +\end{verbatim} + +The hold reason makes things obvious as soon as you know that +\texttt{iwd} stands for "Initial Working Directory". There is no way +that the Execute node \texttt{ipp029} can access the contents of what is +on the \texttt{ippc01} local file system. + +To fix the problem, stop processing with: +\begin{verbatim} +$ condor_rm 71204.0 +\end{verbatim} + +Note the creation of the \texttt{rescue} file in +\texttt{/export/ippc01.0/ippdor/simple\_dag} + +Now submit the DAG from any NFS directory: +\begin{verbatim} +ippdor@ippc01 ~ $ condor_submit_dag /export/ippc01.0/ippdor/simple_dag/AB.dag + +Running rescue DAG 1 +----------------------------------------------------------------------- +File for submitting this DAG to Condor : /export/ippc01.0/ippdor/simple_dag/AB.dag.condor.sub +Log of DAGMan debugging messages : /export/ippc01.0/ippdor/simple_dag/AB.dag.dagman.out +Log of Condor library output : /export/ippc01.0/ippdor/simple_dag/AB.dag.lib.out +Log of Condor library error messages : /export/ippc01.0/ippdor/simple_dag/AB.dag.lib.err +Log of the life of condor_dagman itself : /export/ippc01.0/ippdor/simple_dag/AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71208. +----------------------------------------------------------------------- +\end{verbatim} + +Note that the "rescue DAG 1" is used. + +... and the DAG should terminate correctly from the \texttt{condor\_q} +command and the contents of the \texttt{AB.dag.dagman.out} file: +\begin{verbatim} +[...] +11/25/11 12:13:58 Done Pre Queued Post Ready Un-Ready Failed +11/25/11 12:13:58 === === === === === === === +11/25/11 12:13:58 2 0 0 0 0 0 0 +11/25/11 12:13:58 0 job proc(s) currently held +11/25/11 12:13:58 All jobs Completed! +11/25/11 12:13:58 Note: 0 total job deferrals because of -MaxJobs limit (0) +11/25/11 12:13:58 Note: 0 total job deferrals because of -MaxIdle limit (0) +11/25/11 12:13:58 Note: 0 total job deferrals because of node category throttles +11/25/11 12:13:58 Note: 0 total PRE script deferrals because of -MaxPre limit (0) +11/25/11 12:13:58 Note: 0 total POST script deferrals because of -MaxPost limit (0) +11/25/11 12:13:58 **** condor_scheduniv_exec.71208.0 (condor_DAGMAN) pid 18455 EXITING WITH STATUS 0 +\end{verbatim} Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/design.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/design.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/design.tex (revision 33415) @@ -0,0 +1,4 @@ +\chapter{Software Design} +\label{chap-software} + +TODO Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/heldjobs.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/heldjobs.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/heldjobs.tex (revision 33415) @@ -0,0 +1,66 @@ +\chapter{Annex} + +\section{Held Jobs} +\label{annex-heldjobs} + +\nul\hfill{\scriptsize This example is related to the DAG shown in + section~\ref{sec-oper-tasks}. The A.condor and B.condor jobs are + defined as "echo A" and "echo B" commands.} + +If \texttt{condor\_submit\_dag} is executed from a local directory +(e.g. \textbf{/export/ippc01.0/ippdor/simple_dag}), the jobs will be +indefinitely held by Condor. + +\begin{verbatim} +ippdor@ippc01 /export/ippc01.0/ippdor/simple_dag $ condor_submit_dag AB.dag +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71204. +----------------------------------------------------------------------- +\end{verbatim} + +This can be seen with the +\texttt{condor\_q} command: + +\begin{verbatim} +$ condor_q + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu + ID OWNER SUBMITTED RUN_TIME ST PRI SIZE CMD +71204.0 ippdor 11/25 11:41 0+00:01:27 R 0 12.2 condor_dagman +71205.0 ippdor 11/25 11:42 0+00:00:01 H 0 0.0 echo A + +2 jobs; 0 idle, 1 running, 1 held +\end{verbatim} + +``In-depth'' analysis can be performed with \texttt{condor\_q -analyze} command: + +\begin{verbatim} +$ condor_q -analyze 71205.0 + +-- Submitter: ippc01.ifa.hawaii.edu : <10.10.20.80:46944> : ippc01.ifa.hawaii.edu +--- +71205.000: Request is held. + +Hold reason: Error from slot7@ipp029.ifa.hawaii.edu: Failed to execute +'/bin/echo' with arguments A: Cannot access specified iwd +"/export/ippc01.0/ippdor/simple_dag" (errno=2: 'No such file or directory') +\end{verbatim} + +The hold reason makes things obvious (as soon as you know that +\texttt{iwd} stands for "Initial Working Directory")... There is no +way that the Execute node \texttt{ipp029} can get the contents of what +is on \texttt{ippc01} local file system. + +To fix the problem, stop processing with: +\begin{verbatim} +$ condor_rm 71204.0 +\end{verbatim} + +Note the creation of the \texttt{rescue} file in \texttt{/export/ippc01.0/ippdor/simple\_dag} Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/ippdor.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/ippdor.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/ippdor.tex (revision 33415) @@ -0,0 +1,100 @@ +\documentclass[10pt]{report} + +\usepackage{vmargin} +\setpapersize{USletter} +\setmarginsrb{0.5in}{0.5in}{1.25in}{0.5in}{0pt}{0mm}{0pt}{0mm} +\usepackage{longtable} +\usepackage{hyperref} +\usepackage{makeidx} +\usepackage{enumitem} + +\title{Condor in the IPP} +\author{S.~Chastel\\Version 0.1} +\date{2011-11-23} + +\makeindex + +\newcommand{\CondorSubmit}{\texttt{condor\_submit}} +\newcommand{\CondorSubmitDag}{\texttt{condor\_submit\_dag}} + +\newcommand{\IppPython}[1]{\texttt{\~{ }ippdor/local/share/python/ipp/{#1}}} + +\newcommand{\definition}[3]{% + %#1: What is displayed in the text + %#2: What is displayed in the margin + %#3: Entry in the index + \emph{#1}\marginpar{\scriptsize #2}% + \index{#3}% +} + +\begin{document} +\maketitle + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\chapter*{Introduction} + +This is the documentation detailing how the Condor HTC system is used +to perform processing for the IPP. Note that it is (currently) not +supposed to replace the \texttt{pantasks} scheduling system (mainly +because we choose not to use Condor as a scheduler). + +This documentation consists of five parts: +\begin{enumerate} +\item This introduction; +\item Operational Users Manual (page~\pageref{chap-operation}): its + targets are the operators of the IPP, that is, persons who want to + use the Condor framework to process data for the IPP; +\item Administration Manual (page~\pageref{chap-adm}): for system + administrators. Mainly how to install and upgrade the Condor + framework; +\item Link with the IPP (page~\pageref{chap-ipp}): TODO: Should be + part of Software Organization; +\item Software Organization (page~\pageref{chap-software}): Design, + Classes. How to add new features. +\end{enumerate} + +\emph{Note:} Be aware that this documentation is not a substitute to +the Condor documentation\definition{}{Condor + documentation}{Condor!Documentation} which is available from the +Condor Project Homepage\footnote{when this document was released, it + was: + \href{http://research.cs.wisc.edu/condor/}{http://research.cs.wisc.edu/condor/}}. + +\subsection*{Document History} +\begin{tabular}{|c|c|p{5in}|} + \hline + \textbf{Version} & \textbf{Date} & \textbf{Comment} \\ + \hline + 0.1 & 2011-11-23 & First draft \\ + \hline +\end{tabular} + +\subsection*{Acronyms} +\begin{longtable}{lp{5in}} + IPP & Image Processing Pipeline \\ + IWD & Initial Working Directory \\ + RTCM & Read the Condor Manual \\ +\end{longtable} + +\setcounter{tocdepth}{2} +\tableofcontents + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{operations.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{administration.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{link_ipp.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{design.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\input{annex.tex} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\printindex + +\end{document} Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/link_ipp.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/link_ipp.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/link_ipp.tex (revision 33415) @@ -0,0 +1,4 @@ +\chapter{Link with the IPP} +\label{chap-ipp} + +TODO Index: branches/meh_branches/ppstack_test/ippdor/doc/Full/operations.tex =================================================================== --- branches/meh_branches/ppstack_test/ippdor/doc/Full/operations.tex (revision 33415) +++ branches/meh_branches/ppstack_test/ippdor/doc/Full/operations.tex (revision 33415) @@ -0,0 +1,622 @@ +\newcommand{\QueueExposuresWarping}{% + \texttt{queue\_exposures\_for\_warping.py}} +\newcommand{\QueueExposuresStacking}{% + \texttt{queue\_exposures\_for\_stacking.py}} + +\chapter{Operators Manual} +\label{chap-operation} + +\section{Condor Framework Availability on the IPP Production Cluster} + +The Condor framework is not available on all hosts of the IPP +production cluster. The purpose of this section is to give some +details about the different nodes which are used and their roles in +Condor. The set of all machines on which Condor is installed is called +the \definition{Condor pool}{Condor pool}{Condor!Pool}. + +The Condor \definition{central manager}{Central + manager}{Condor!Central manager} is \texttt{ippdb03}. From the +Condor documentation (section~3.1), the role of the central manager is +to "[collect] information, and [negotiate] between resources and +resource requests". It "plays a very important part in the Condor pool +and should be reliable" since "[if it] crashes, no further matchmaking +can be performed within the Condor system". Moreover "[all] queries go +to the central manager". + +The other nodes of the IPP production cluster where Condor is +installed are all both Execute and Submit Condor nodes. The complete +list of those nodes is available in the file +\texttt{\~{ }ippdor/local/share/CondorNodes}. Note that a node on which +Condor is installed is not necessary available for processing in +Condor since it can be deactivated (see section~\ref{sec-oper-hosts} +for details). + +An \definition{Execute}{Execute node}{Condor!Execute node} node is a +node where processing can be performed. From the Condor manual +section~3.1, "the only resource that might matter is disk +space"... Until now, it has never been a matter. + +A \definition{Submit}{Submit node}{Condor!Submit node} node is a node +where Condor jobs can be submitted (see +section~\ref{sec-ippdor-run}). It is suggested (again this is from the +Condor manual, section~3.1), that since "every job that you submit +that is currently running on a remote machine generates another +process on your submit machine [, ...] if you have lots of jobs +running, you will need a fair amount of swap space and/or real +memory". Until now, jobs were generally submitted from one of the +\texttt{ippc\emph{xy}} nodes. + +Note: For Condor gurus, there is no Checkpoint Server since all +processing is performed in the \emph{vanilla} Condor universe. + +\section{Condor Dedicated User: \texttt{ippdor}} +\label{sec-oper-login} + +To use the Condor framework you need to log into the IPP production +cluster as the user \texttt{ippdor}. Note that since Condor is not +installed on \texttt{ippc18}, you will have to login again into +another node for the condor commands to be usable. \texttt{ipp004} or +\texttt{ippdb03} are other "open" nodes where you can connect. + +Note: The \texttt{ippdor} account uses \texttt{bash}. + +Theoretically it should be possible to use the Condor framework with +your own login but 1) It has not been tested; 2) No support will be +provided for that; 3) So... RTCM. + +As detailed in section~\ref{sec-adm-ippdor}, the \texttt{ippdor} +account uses files that may be localized on the Condor node. If you +get a \texttt{command not found} error, make sure you are on one of +the node where Condor is installed\footnote{that is, the command + \texttt{"grep \`{ }hostname\`{ } \~{ + }ippdor/local/share/CondorNodes"} should display the host name. If + you don't see anything, read the first paragraph of this section.}. + +\section{Managing Condor Execution on the IPP Production Cluster + Nodes} +\label{sec-oper-hosts} + +Note 1: To install Condor on a new node, see +section~\ref{sec-adm-newnode}. + +Note 2: Except for a few home-grown commands, information was +extracted from the Condor Manual section~3.10. + +\subsection{Node Availability} +\label{sec-oper-hosts-availability} +\definition{}{Node availability}{Node availability}% +\definition{}{}{Condor!Node availability}% +To check if a node can be used by Condor, check the result of the +\texttt{ipp\_condor\_status } command. You should get some +information about the host from the Condor point of view. For +instance, \texttt{ipp060} potentially has 12 potential processors for +Condor execution\footnote{To get more information about what the + columns mean, RTCM}: +\begin{verbatim} +ippdor@ippc20 ~ $ ipp_condor_status ipp060 +Name OpSys Arch State Activity LoadAv Mem ActvtyTime + +slot10@ipp060.ifa. LINUX X86_64 Unclaimed Idle 0.000 4032 0+05:54:29 +slot11@ipp060.ifa. LINUX X86_64 Owner Idle 0.400 4032 0+00:00:06 +slot12@ipp060.ifa. LINUX X86_64 Owner Idle 1.000 4032 0+00:00:07 +slot1@ipp060.ifa.h LINUX X86_64 Owner Idle 0.840 4032 0+00:00:04 +slot2@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:34:09 +slot3@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+06:56:32 +slot4@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:25:44 +slot5@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+14:33:57 +slot6@ipp060.ifa.h LINUX X86_64 Claimed Idle 0.000 4032 3+11:22:07 +slot7@ipp060.ifa.h LINUX X86_64 Owner Idle 1.000 4032 0+00:00:29 +slot8@ipp060.ifa.h LINUX X86_64 Owner Idle 0.790 4032 0+00:00:03 +slot9@ipp060.ifa.h LINUX X86_64 Unclaimed Idle 0.000 4032 0+06:55:39 + Total Owner Claimed Unclaimed Matched Preempting Backfill + + X86_64/LINUX 12 5 1 6 0 0 0 + + Total 12 5 1 6 0 0 0 +\end{verbatim} + +If you don't get anything, the node has been deactivated for Condor +processing). For instance (while the author writes these lines), +\texttt{ipp065} will not execute any Condor jobs: +\begin{verbatim} +ippdor@ippc20 ~ $ ipp_condor_status ipp065 +ippdor@ippc20 ~ $ +\end{verbatim} + +\emph{Note:} \texttt{ipp\_condor\_status} is an alias of +\texttt{condor\_status} defined in \texttt{\~{ }ippdor/.bashrc}. + +\subsection{Make a Node Unavailable} +\label{sec-oper-hosts-remove} + +To remove a node from the list of execution nodes, execute the command +\texttt{ipp\_condor\_off } from any node where Condor is +installed. No job will be started/executed and submitted from that +host. \texttt{ipp\_condor\_off} is a function defined in \texttt{\~{ + }ippdor/.bashrc}. + +To stop Condor execution and submission from \definition{all}{Stopping + Condor on stare nodes}{Stopping Condor on stare nodes}stare nodes, +the \texttt{stare\_nodes\_off} and \texttt{stare\_nodes\_on} scripts +are provided. As it can be expected, those scripts just call +\texttt{ipp\_condor\_off} or \texttt{ipp\_condor\_on} for each of the +stares nodes. + +\subsection{Make a Node Available} +\label{sec-oper-hosts-available} + +To make a node available for Condor after it has been stopped, execute +the command \texttt{ipp\_condor\_on } from any node where +Condor is installed. + +\emph{Note:} If the following message is displayed, +\begin{verbatim} +bash: condor_master: command not found +\end{verbatim} +it simply means that Condor is not installed on the node where you try +to start it. + +\section{Condor Jobs and DAG} + +\subsection{Running Jobs in Condor} +\label{sec-oper-jobs} +We will call \definition{job}{job}{job} an elementary unit of +processing. It consists of the execution of one instance of an +executable (we will assume that an executable is a program that can be +called directly from the command line). For instance, "/bin/echo One" +and "/bin/echo Two" are two jobs, each job being an instance of the +same executable "/bin/echo". + +In our case, defining a correct Condor job is as simple (not to say +simplistic) as defining: +\begin{itemize} +\item The name of the executable; +\item The Condor universe of execution (in the IPP case, to avoid + recompilation of programs we use the \emph{vanilla} universe); +\item The file where the standard output of the executable will go (on + an NFS directory since we don't know where the executable will be + run); +\item The file where the error output of the executable will go (on + an NFS directory since we don't know where the executable will be + run); +\item The file where Condor will write the log of the job (on a local + directory); +\item The requirements about the node where execution will take place; +\item Some argument to tell that we want to keep the environment + variables; +\item The arguments for each execution of the executable. +\end{itemize} + +With the example given on \texttt{/bin/echo}, the contents of the +condor job file (let's call this file \texttt{echo\_test.condor}) are: +\begin{verbatim} +Executable = /bin/echo +Universe = vanilla +Output = /home/panstarrs/ippdor/tmp/tests/out.echo_test +Error = /home/panstarrs/ippdor/tmp/tests/err.echo_test +Log = /tmp/echo_test.log +Requirements = LoadAvg < 0.3 + +getenv = True + +Arguments = One +Queue + +Arguments = Two +Queue +\end{verbatim} + +To execute those jobs, they just need to be \definition{submitted}{Job + submission}{Job submission} from any Submit node to the Condor +system with: + +\texttt{condor\_submit echo\_test.condor} + +The jobs are (usually) not executed immediately and there is no way of +guessing the order on the jobs execution (if the order of jobs +execution is a matter, look at paragraph~\ref{sec-oper-tasks}). + +Note: From what the author experienced, \texttt{condor\_submit} and +the condor jobs definition file can be executed and placed on any kind +of file system (local and/or NFS) which is different from +\texttt{condor\_submit\_dag}. + +\subsection{Running tasks} +\label{sec-oper-tasks} + +\subsubsection{DAGMan} + +We will call \definition{task}{task}{task} a set of jobs where jobs +chaining depend on each other. Condor provides a mechanism of +organizing jobs according to their dependencies: +\definition{DAGMan}{DAGMan}{DAGMan} (which stands for Directed Acyclic +Graph Manager). Exhaustive documentation about DAGMan can be found in +the Condor documentation section 2.10. In this section I will just +give details about what is used for the IPP. + +\subsubsection{DAG Description File} + +To make it simple, a DAG is a high-level description of how jobs are +sequenced. If the job B has to be performed after the job A, the +dependency is formalized this way: +\begin{verbatim} +JOB A /some/directory/A.condor +JOB B /some/directory/B.condor +PARENT A CHILD B +\end{verbatim} + +Assuming the previous text is saved in the file named \texttt{AB.dag}, +the execution of the DAG can be \definition{submitted}{DAG + submission}{DAG submission} from any Submit node to the Condor +system using: + +\texttt{condor\_submit\_dag AB.dag} + +The output of that command will look like: +\begin{verbatim} +----------------------------------------------------------------------- +File for submitting this DAG to Condor : AB.dag.condor.sub +Log of DAGMan debugging messages : AB.dag.dagman.out +Log of Condor library output : AB.dag.lib.out +Log of Condor library error messages : AB.dag.lib.err +Log of the life of condor_dagman itself : AB.dag.dagman.log + +Submitting job(s). +1 job(s) submitted to cluster 71201. +----------------------------------------------------------------------- +\end{verbatim} + +All those files are created in the directory of the \texttt{AB.dag} +file. The \texttt{AB.dag.dagman.out} file is the most interesting log +file. + +\subsubsection{Files and Directories Locations} + +The DAG file can be localized on any file system: local or +NFS. However for performance concerns, I recommend to put it on the +local file system of the submit node. Monitor to the log growth rate +(the logs are not huge though... For instance, full processing of 2300 +MD06 exposures from Chip to Warp stages in the IPP created a log of 15 +MB for the \texttt{.out} file and of a total of about 50 kB for the 4 +other files). + +The \texttt{/some/directory} directory (where the condor jobs are +defined) can be placed on any file system. Again, I suggest to put +them on the local file system of the Submit node (mostly for +performance concerns). + +\begin{center} + \fbox{\parbox{6in}{\textbf{Important note:} The + \texttt{condor\_submit\_dag} command \textbf{must} be executed + \textbf{from a NFS file system} or the jobs will be indefinitely + \texttt{Held} (shown with \texttt{H} status in + \texttt{condor\_q}. See section~\ref{annex-heldjobs} for more + details.}} +\end{center} + +One restriction is that there cannot be cycles, i.e. circular +dependencies. For instance, job Egg depends on job Chicken, which +depend on job Egg. If you submit a DAG with a cycle, looking at the +\texttt{dagman.out} file, you will see a message like: +\begin{verbatim} +[...] +11/25/11 12:22:02 ERROR: DAG finished but not all nodes are complete -- + checking for a cycle... +11/25/11 12:22:02 ... ERROR: a cycle exists in the dag, plese check input +[...] +11/25/11 12:22:03 **** condor_scheduniv_exec.71211.0 (condor_DAGMAN) + pid 20996 EXITING WITH STATUS 1 +\end{verbatim} + +\subsubsection{Advanced DAGMan Features} +\label{sec-oper-dagman} + +DAGMan offers other convenient functions for a DAG description. As for +the other Condor features the reader is invited to check the Condor +documentation for in-depth details about them. + +\paragraph{SubDAG:} A graph node of a DAG can be itself a DAG making +the development and the analysis of a processing chain easier. A DAG +in a DAG is called a \definition{SubDAG}{SubDAG}{SubDAG} and can be +added to a DAG using this syntax: +\begin{verbatim} +SUBDAG EXTERNAL +\end{verbatim} + +\paragraph{Script:} Scripts canbe executed before or after a node +execution. They can be added using the following syntax: +\begin{verbatim} +SCRIPT PRE [] +SCRIPT POST [] +\end{verbatim} + +From the Condor documentation, "the \texttt{} specifies the +node to which the script is attached. The \texttt{} +specifies the script to be executed, and it may be followed by any +command line \texttt{} to that script". In the IPP context +we currently use only the \texttt{SCRIPT POST} feature. + +\subsection{Condor Interrupted} +\label{sec-oper-condor-interrupted} + +If the execution of Condor DAG is interrupted, for instance, because +of an operator \texttt{condor\_rm} or because of errors, Condor marks +the jobs or subdags that were successfully completed by adding the +\texttt{DONE} keyword to the DAG file. The resulting DAG file is saved +in a \definition{rescue DAG file}{Rescue DAG file}{Rescue DAG file}, +which is simply a copy of the original DAG file with a +\texttt{rescue} file extension (\texttt{} is +an auto-incrementing integer coded on three digits, starting at +\texttt{001}). + +To \definition{rescue processing}{Rescuing processing}{Rescuing + processing} you just need to execute \textbf{exactly} the same +submission command: Condor will use the last rescue file that was +produced. + +Note: If you know what you are doing you can create you rown rescue +file or add/remove \texttt{DONE} keywords to/from a rescue file. Some +utility programs can do that for you for some IPP stages +(see~\ref{TODO}). + +\section{IPP Tasks} +\label{sec-oper-ipp} + +The IPP defines complex tasks. The purpose of this section is to +provide a user manual about the programs that need to be run to +prepare the set of Condor files which are needed to perform some of +the IPP tasks. Developers should look at chapter~\ref{chap-software} +for in-depth details. + +Each subsection of this section details: 1) The software role; 2) Its +required inputs; 3) Its outputs; 4) Its IPP products. + +Currently the IPP parts that are implemented are: +\begin{itemize}% + \setlength{\itemsep}{0pt}\setlength{\topsep}{0pt}\setlength{\partopsep}{0pt}\setlength{\parsep}{0pt}% +\item Chip to Warp stages (paragraph~\ref{sec-oper-ipp-chip2warp}); +\item Chip to Stack stages (paragraph~\ref{sec-oper-ipp-chip2stack}); +\end{itemize} + +\subsection{Chip to Warp Processing: \QueueExposuresWarping{}} +\label{sec-oper-ipp-chip2warp} + +Since this section is supposed to belong to the operators manual, no +detail about the software will be provided. The developer will read +section~\ref{sec-TODO} for such details. + +\subsubsection{Role} +The role of \QueueExposuresWarping{} is to prepare the set of Condor +and IPP elements which are required to process a set of exposures from +the Chip to the Warp stage. + +The \QueueExposuresWarping{} script has four roles: +\begin{enumerate}[topsep=0pt, partopsep=0pt, itemsep=0pt, parsep=0pt, itemindent=10pt] +\item Assert that the label has not already been used and that the + input exposures exist; +\item Create entries in the \texttt{chipRun} table; +\item Create the DAG files which are necessary for the processing of + all exposures; +\item Display the \texttt{submit\_condor\_dag} command that needs to + be run by the operator. +\end{enumerate} + +Note that the operator has to run the \texttt{submit\_condor\_dag} +command. + +\subsubsection{Submit Node} + +A node needs to be chosen as Submit node. TODO: add details. + +\subsubsection{Inputs} +The inputs are: +\begin{itemize} +\item A \textbf{label} The label used for processing. If the label has + already been used, the program is aborted. As for the processing in + IPP, don't use spaces or non-ASCII characters for the label (for + instance, \texttt{My.New\_Label}); +\item An \textbf{e-mail address} who gets the e-mails (for instance, + \texttt{someone@mail.com}); +\item A \textbf{tessellation ID} (e.g. \texttt{MD04.V3}); +\item A \textbf{list of exposures} to be processed (for instance, + \texttt{o4926g0064o}, \texttt{o4926g0068o}, \texttt{o4926g0069o}, + \texttt{o4926g0071o}, and \texttt{o4940g0237o}). +\end{itemize} + +The \QueueExposuresWarping{} script can be called in two ways: +\begin{itemize} +\item Either they are provided in a file which has the format of a + Properties-style file. With the previous parameters: +\begin{verbatim} +label = My.New_Label +tessId = MD04.V3 +recipient = someone@mail.com +exposure = o4926g0064o +exposure = o4926g0068o +exposure = o4926g0069o +exposure = o4926g0071o +# Comment line +exposure = o4940g0237o # Line followed by a comment +\end{verbatim} + Assume that the file is stored under the name + \texttt{~{ }ippdor/examples/ChipsToWarps.ippdor} + the script is called with (note the \texttt{-f} option flag): +\begin{verbatim} +queue_exposures_for_warping.py -f ~ippdor/examples/01/ChipsToWarps.ippdor +\end{verbatim} +\item Or the input parameters are provided as arguments, that is (the + arguments have to be in this order): +\begin{verbatim} +queue_exposures_for_warping.py My.New_Label MD04.V3 someone@mail.com \ + o4926g0064o o4926g0068o o4926g0069o o4926g0071o o4940g0237o +\end{verbatim} +\end{itemize} + +\subsubsection{Outputs. Entries on GPC1 Database} +The script writes its outputs to the directory named +\texttt{/
No imageNo imageNo imageNo image
No imageNo imageNo image
Update UpperRight
Update UpperRight
Problem in Upper part!
Problem in Upper part!
Update UpperRight
Problem in Upper part!
4z
5y
6z
1-999
2-999
3-999
4-999
5-999
6-999
7-999
8-999
9-999
10-999
11-999
12-999
13-999
14-999
15-999
16-999
17-999
18-999
19-999
20-999
21-999
22-999
23-999
24-999
25-999
26-999
27-999
28-999
29-999
30-999
31-999
32-999
33-999
34-999
35-999
36-999
37-999
38-999
39-999
40-999
41-999
42-999
43-999
44-999
45-999
46-999
47-999
48-999
49-999
50-999
51-999
52-999
53-999
54-999
55-999
56-999
57-999
58-999
59-999
60-999
61-999
62-999
63-999
64-999
65-999
66-999
67-999
68-999
69-999
70-999
71-999
72-999
73-999
74-999
75-999
76-999
77-999
78-999
79-999
80-999
81-999
82-999
83-999
84-999
85-999
86-999
87-999
88-999
89-999
90-999
91-999
92-999
93-999
94-999
95-999
96-999
97-999
98-999
99-999
100-999
101-999
102-999
103-999
104-999
105-999
106-999
107-999
108-999
109-999
110-999
111-999
112-999
113-999
114-999
115-999
116-999
117-999
118-999
119-999
120-999
121-999
122-999
123-999
124-999
125-999
126-999
127-999
128-999
129-999
130-999
131-999
132-999
133-999
134-999
135-999
136-999
137-999
138-999
139-999
140-999
141-999
142-999
143-999
144-999
145-999
146-999
147-999
148-999
149-999
150-999
151-999
152-999
153-999
154-999
155-999
156-999
157-999
158-999
159-999
160-999
161-999
162-999
163-999
164-999
165-999
166-999
167-999
168-999
169-999
170-999
171-999
172-999
173-999
174-999
175-999
176-999
177-999
178-999
179-999
180-999
181-999
182-999
183-999
184-999
185-999
186-999
187-999
188-999
189-999
190-999
191-999
192-999
193-999
194-999
195-999
196-999
197-999
198-999
199-999
200-999
201-999
202-999
203-999
204-999
205-999
206-999
207-999
208-999
209-999
210-999
211-999
212-999
213-999
214-999
215-999
216-999
217-999
218-999
219-999
220-999
221-999
222-999
223-999
224-999
225-999
226-999
227-999
228-999
229-999
230-999
231-999
232-999
233-999
234-999
235-999
236-999
237-999
238-999
239-999
240-999
241-999
242-999
243-999
244-999
245-999
246-999
247-999
248-999
249-999
250-999
251-999
252-999
253-999
254-999
255-999
256-999
257-999
258-999
259-999
260-999
261-999
262-999
263-999
264-999
265-999
266-999
267-999
268-999
269-999
270-999
271-999
272-999
273-999
274-999
275-999
276-999
277-999
278-999
279-999
280-999
281-999
282-999
283-999
284-999
285-999
286-999
287-999
288-999
289-999
290-999
291-999
292-999
293-999
294-999
295-999
296-999
297-999
298-999
299-999
300-999
301-999
302-999
303-999
304-999
305-999
306-999
307-999
308-999
309-999
310-999
311-999
312-999
313-999
314-999
315-999
316-999
317-999
318-999
319-999
320-999
321-999
322-999
323-999
324-999
325-999
326-999
327-999
328-999
329-999
330-999
331-999
332-999
333-999
334-999
335-999
336-999
337-999
338-999
339-999
340-999
341-999
342-999
343-999
344-999
345-999
346-999
347-999
348-999
349-999
350-999
351-999
352-999
353-999
354-999
355-999
356-999
357-999
358-999
359-999
360-999
361-999
362-999
363-999
364-999
365-999
366-999
367-999
368-999
369-999
370-999
371-999
372-999
373-999
374-999
375-999
376-999
377-999
378-999
379-999
380-999
381-999
382-999
383-999
384-999
385-999
386-999
387-999
388-999
389-999
390-999
391-999
392-999
393-999
394-999
395-999
396-999
397-999
398-999
399-999
400-999
401-999
402-999
403-999
404-999
405-999
406-999
407-999
408-999
409-999
410-999
411-999
412-999
413-999
414-999
415-999
416-999
417-999
418-999
419-999
420-999
421-999
422-999
423-999
424-999
425-999
426-999
427-999
428-999
429-999
430-999
431-999
432-999
433-999
434-999
435-999
436-999
437-999
438-999
439-999
440-999
441-999
442-999
443-999
444-999
445-999
446-999
447-999
448-999
449-999
450-999
451-999
452-999
453-999
454-999
455-999
456-999
457-999
458-999
459-999
460-999
461-999
462-999
463-999
464-999
465-999
466-999
467-999
468-999
469-999
470-999
471-999
472-999
473-999
474-999
475-999
476-999
477-999
478-999
479-999
480-999
481-999
482-999
483-999
484-999
485-999
486-999
487-999
488-999
489-999
490-999
491-999
492-999
493-999
494-999
495-999
496-999
497-999
498-999
499-999
500-999
501-999
502-999
503-999
504-999
505-999
506-999
507-999
508-999
509-999
510-999
511-999
512-999
513-999
514-999
515-999
516-999
517-999
518-999
519-999
520-999
521-999
522-999
523-999
524-999
525-999
526-999
527-999
528-999
529-999
530-999
531-999
532-999
533-999
534-999
535-999
536-999
537-999
538-999
539-999
540-999
541-999
542-999
543-999
544-999
545-999
546-999
547-999
548-999
549-999
550-999
551-999
552-999
553-999
554-999
555-999
556-999
557-999
558-999
559-999
560-999
561-999
562-999
563-999
564-999
565-999
566-999
567-999
568-999
569-999
570-999
571-999
572-999
573-999
574-999
575-999
576-999
577-999
578-999
579-999
580-999
581-999
582-999
583-999
584-999
585-999
586-999
587-999
588-999
589-999
590-999
591-999
592-999
593-999
594-999
595-999
596-999
597-999
598-999
599-999
600-999
601-999
602-999
603-999
604-999
605-999
606-999
607-999
608-999
609-999
610-999
611-999
612-999
613-999
614-999
615-999
616-999
617-999
618-999
619-999
620-999
621-999
622-999
623-999
624-999
625-999
626-999
627-999
628-999
629-999
630-999
631-999
632-999
633-999
634-999
635-999
636-999
637-999
638-999
639-999
640-999
641-999
642-999
643-999
644-999
645-999
646-999
647-999
648-999
649-999
650-999
651-999
652-999
653-999
654-999
655-999
656-999
657-999
658-999
659-999
660-999
661-999
662-999
663-999
664-999
665-999
666-999
667-999
668-999
669-999
670-999
671-999
672-999
673-999
674-999
675-999
676-999
677-999
678-999
679-999
680-999
681-999
682-999
683-999
684-999
685-999
686-999
687-999
688-999
689-999
690-999
691-999
692-999
693-999
694-999
695-999
696-999
697-999
698-999
699-999
700-999
701-999
702-999
703-999
704-999
705-999
706-999
707-999
708-999
709-999
710-999
711-999
712-999
713-999
714-999
715-999
716-999
717-999
718-999
719-999
720-999
721-999
722-999
723-999
724-999
725-999
726-999
727-999
728-999
729-999
730-999
731-999
732-999
733-999
734-999
735-999
736-999
737-999
738-999
739-999
740-999
741-999
742-999
743-999
744-999
745-999
746-999
747-999
748-999
749-999
750-999
751-999
752-999
753-999
754-999
755-999
756-999
757-999
758-999
759-999
760-999
761-999
762-999
763-999
764-999
765-999
766-999
767-999
768-999
769-999
770-999
771-999
772-999
773-999
774-999
775-999
776-999
777-999
778-999
779-999
780-999
781-999
782-999
783-999
784-999
785-999
786-999
787-999
788-999
789-999
790-999
791-999
792-999
793-999
794-999
795-999
796-999
797-999
798-999
799-999
800-999
801-999
802-999
803-999
804-999
805-999
806-999
807-999
808-999
809-999
810-999
811-999
812-999
813-999
814-999
815-999
816-999
817-999
818-999
819-999
820-999
821-999
822-999
823-999
824-999
825-999
826-999
827-999
828-999
829-999
830-999
831-999
832-999
833-999
834-999
835-999
836-999
837-999
838-999
839-999
840-999
841-999
842-999
843-999
844-999
845-999
846-999
847-999
848-999
849-999
850-999
851-999
852-999
853-999
854-999
855-999
856-999
857-999
858-999
859-999
860-999
861-999
862-999
863-999
864-999
865-999
866-999
867-999
868-999
869-999
870-999
871-999
872-999
873-999
874-999
875-999
876-999
877-999
878-999
879-999
880-999
881-999
882-999
883-999
884-999
885-999
886-999
887-999
888-999
889-999
890-999
891-999
892-999
893-999
894-999
895-999
896-999
897-999
898-999
899-999
900-999
901-999
902-999
903-999
904-999
905-999
906-999
907-999
908-999
909-999
910-999
911-999
912-999
913-999
914-999
915-999
916-999
917-999
918-999
919-999
920-999
921-999
922-999
923-999
924-999
925-999
926-999
927-999
928-999
929-999
930-999
931-999
932-999
933-999
934-999
935-999
936-999
937-999
938-999
939-999
940-999
941-999
942-999
943-999
944-999
945-999
946-999
947-999
948-999
949-999
950-999
951-999
952-999
953-999
954-999
955-999
956-999
957-999
958-999
959-999
960-999
961-999
962-999
963-999
964-999
965-999
966-999
967-999
968-999
969-999
970-999
971-999
972-999
973-999
974-999
975-999
976-999
977-999
978-999
979-999
980-999
981-999
982-999
983-999
984-999
985-999
986-999
987-999
988-999
989-999
990-999
991-999
992-999
993-999
994-999
995-999
996-999
997-999
998-999
999-999
1000-999
1001-999
1002-999
1003-999
1004-999
1005-999
1006-999
1007-999
1008-999
1009-999
1010-999
1011-999
1012-999
1013-999
1014-999
1015-999
1016-999
1017-999
1018-999
1019-999
1020-999
1021-999
1022-999
1023-999
1024-999
1025-999
1026-999
1027-999
1028-999
1029-999
1030-999
1031-999
1032-999
1033-999
1034-999
1035-999
1036-999
1037-999
1038-999
1039-999
1040-999
1041-999
1042-999
1043-999
1044-999
1045-999
1046-999
1047-999
1048-999
1049-999
1050-999
1051-999
1052-999
1053-999
1054-999
1055-999
1056-999
1057-999
1058-999
1059-999
1060-999
1061-999
1062-999
1063-999
1064-999
1065-999
1066-999
1067-999
1068-999
1069-999
1070-999
1071-999
1072-999
1073-999
1074-999
1075-999
1076-999
1077-999
1078-999
1079-999
1080-999
1081-999
1082-999
1083-999
1084-999
1085-999
1086-999
1087-999
1088-999
1089-999
1090-999
1091-999
1092-999
1093-999
1094-999
1095-999
1096-999
1097-999
1098-999
1099-999
1100-999
1101-999
1102-999
1103-999
1104-999
1105-999
1106-999
1107-999
1108-999
1109-999
1110-999
1111-999
1112-999
1113-999
1114-999
1115-999
1116-999
1117-999
1118-999
1119-999
1120-999
1121-999
1122-999
1123-999
1124-999
1125-999
1126-999
1127-999
1128-999
1129-999
1130-999
1131-999
1132-999
1133-999
1134-999
1135-999
1136-999
1137-999
1138-999
1139-999
1140-999
1141-999
1142-999
1143-999
1144-999
1145-999
1146-999
1147-999
1148-999
1149-999
1150-999
1151-999
1152-999
1153-999
1154-999
1155-999
1156-999
1157-999
1158-999
1159-999
1160-999
1161-999
1162-999
1163-999
1164-999
1165-999
1166-999
1167-999
1168-999
1169-999
1170-999
1171-999
1172-999
1173-999
1174-999
1175-999
1176-999
1177-999
1178-999
1179-999
1180-999
1181-999
1182-999
1183-999
1184-999
1185-999
1186-999
1187-999
1188-999
1189-999
1190-999
1191-999
1192-999
1193-999
1194-999
1195-999
1196-999
1197-999
1198-999
1199-999
1200-999
1201-999
1202-999
1203-999
1204-999
1205-999
1206-999
1207-999
1208-999
1209-999
1210-999
1211-999
1212-999
1213-999
1214-999
1215-999
1216-999
1217-999
1218-999
1219-999
1220-999
1221-999
1222-999
1223-999
1224-999
1225-999
1226-999
1227-999
1228-999
1229-999
1230-999
1231-999
1232-999
1233-999
1234-999
1235-999
1236-999
1237-999
1238-999
1239-999
1240-999
1241-999
1242-999
1243-999
1244-999
1245-999
1246-999
1247-999
1248-999
1249-999
1250-999
1251-999
1252-999
1253-999
1254-999
1255-999
1256-999
1257-999
1258-999
1259-999
1260-999
1261-999
1262-999
1263-999
1264-999
1265-999
1266-999
1267-999
1268-999
1269-999
1270-999
1271-999
1272-999
1273-999
1274-999
1275-999
1276-999
1277-999
1278-999
1279-999
1280-999
1281-999
1282-999
1283-999
1284-999
1285-999
1286-999
1287-999
1288-999
1289-999
1290-999
1291-999
1292-999
1293-999
1294-999
1295-999
1296-999
1297-999
1298-999
1299-999
1300-999
1301-999
1302-999
1303-999
1304-999
1305-999
1306-999
1307-999
1308-999
1309-999
1310-999
1311-999
1312-999
1313-999
1314-999
1315-999
1316-999
1317-999
1318-999
1319-999
1320-999
1321-999
1322-999
1323-999
1324-999
1325-999
1326-999
1327-999
1328-999
1329-999
1330-999
1331-999
1332-999
1333-999
1334-999
1335-999
1336-999
1337-999
1338-999
1339-999
1340-999
1341-999
1342-999
1343-999
1344-999
1345-999
1346-999
1347-999
1348-999
1349-999
1350-999
1351-999
1352-999
1353-999
1354-999
1355-999
1356-999
1357-999
1358-999
1359-999
1360-999
1361-999
1362-999
1363-999
1364-999
1365-999
1366-999
1367-999
1368-999
1369-999
1370-999
1371-999
1372-999
1373-999
1374-999
1375-999
1376-999
1377-999
1378-999
1379-999
1380-999
1381-999
1382-999
1383-999
1384-999
1385-999
1386-999
1387-999
1388-999
1389-999
1390-999
1391-999
1392-999
1393-999
1394-999
1395-999
1396-999
1397-999
1398-999
1399-999
1400-999
1401-999
1402-999
1403-999
1404-999
1405-999
1406-999
1407-999
1408-999
1409-999
1410-999
1411-999
1412-999
1413-999
1414-999
1415-999
1416-999
1417-999
1418-999
1419-999
1420-999
1421-999
1422-999
1423-999
1424-999
1425-999
1426-999
1427-999
1428-999
1429-999
1430-999
1431-999
1432-999
1433-999
1434-999
1435-999
1436-999
1437-999
1438-999
1439-999
1440-999
1441-999
1442-999
1443-999
1444-999
1445-999
1446-999
1447-999
1448-999
1449-999
1450-999
1451-999
1452-999
1453-999
1454-999
1455-999
1456-999
1457-999
1458-999
1459-999
1460-999
1461-999
1462-999
1463-999
1464-999
1465-999
1466-999
1467-999
1468-999
1469-999
1470-999
1471-999
1472-999
1473-999
1474-999
1475-999
1476-999
1477-999
1478-999
1479-999
1480-999
1481-999
1482-999
1483-999
1484-999
1485-999
1486-999
1487-999
1488-999
1489-999
1490-999
1491-999
1492-999
1493-999
1494-999
1495-999
1496-999
1497-999
1498-999
1499-999
1500-999
1501-999
1502-999
1503-999
1504-999
1505-999
1506-999
1507-999
1508-999
1509-999
1510-999
1511-999
1512-999
1513-999
1514-999
1515-999
1516-999
1517-999
1518-999
1519-999
1520-999
1521-999
1522-999
1523-999
1524-999
1525-999
1526-999
1527-999
1528-999
1529-999
1530-999
1531-999
1532-999
1533-999
1534-999
1535-999
1536-999
1537-999
1538-999
1539-999
1540-999
1541-999
1542-999
1543-999
1544-999
1545-999
1546-999
1547-999
1548-999
1549-999
1550-999
1551-999
1552-999
1553-999
1554-999
1555-999
1556-999
1557-999
1558-999
1559-999
1560-999
1561-999
1562-999
1563-999
1564-999
1565-999
1566-999
1567-999
1568-999
1569-999
1570-999
1571-999
1572-999
1573-999
1574-999
1575-999
1576-999
1577-999
1578-999
1579-999
1580-999
1581-999
1582-999
1583-999
1584-999
1585-999
1586-999
1587-999
1588-999
1589-999
1590-999
1591-999
1592-999
1593-999
1594-999
1595-999
1596-999
1597-999
1598-999
1599-999
1600-999
1601-999
1602-999
1603-999
1604-999
1605-999
1606-999
1607-999
1608-999
1609-999
1610-999
1611-999
1612-999
1613-999
1614-999
1615-999
1616-999
1617-999
1618-999
1619-999
1620-999
1621-999
1622-999
1623-999
1624-999
1625-999
1626-999
1627-999
1628-999
1629-999
1630-999
1631-999
1632-999
1633-999
1634-999
1635-999
1636-999
1637-999
1638-999
1639-999
1640-999
1641-999
1642-999
1643-999
1644-999
1645-999
1646-999
1647-999
1648-999
1649-999
1650-999
1651-999
1652-999
1653-999
1654-999
1655-999
1656-999
1657-999
1658-999
1659-999
1660-999
1661-999
1662-999
1663-999
1664-999
1665-999
1666-999
1667-999
1668-999
1669-999
1670-999
1671-999
1672-999
1673-999
1674-999
1675-999
1676-999
1677-999
1678-999
1679-999
1680-999
1681-999
1682-999
1683-999
1684-999
1685-999
1686-999
1687-999
1688-999
1689-999
1690-999
1691-999
1692-999
1693-999
1694-999
1695-999
1696-999
1697-999
1698-999
1699-999
1700-999
1701-999
1702-999
1703-999
1704-999
1705-999
1706-999
1707-999
1708-999
1709-999
1710-999
1711-999
1712-999
1713-999
1714-999
1715-999
1716-999
1717-999
1718-999
1719-999
1720-999
1721-999
1722-999
1723-999
1724-999
1725-999
1726-999
1727-999
1728-999
1729-999
1730-999
1731-999
1732-999
1733-999
1734-999
1735-999
1736-999
1737-999
1738-999
1739-999
1740-999
1741-999
1742-999
1743-999
1744-999
1745-999
1746-999
1747-999
1748-999
1749-999
1750-999
1751-999
1752-999
1753-999
1754-999
1755-999
1756-999
1757-999
1758-999
1759-999
1760-999
1761-999
1762-999
1763-999
1764-999
1765-999
1766-999
1767-999
1768-999
1769-999
1770-999
1771-999
1772-999
1773-999
1774-999
1775-999
1776-999
1777-999
1778-999
1779-999
1780-999
1781-999
1782-999
1783-999
1784-999
1785-999
1786-999
1787-999
1788-999
1789-999
1790-999
1791-999
1792-999
1793-999
1794-999
1795-999
1796-999
1797-999
1798-999
1799-999
1800-999
1801-999
1802-999
1803-999
1804-999
1805-999
1806-999
1807-999
1808-999
1809-999
1810-999
1811-999
1812-999
1813-999
1814-999
1815-999
1816-999
1817-999
1818-999
1819-999
1820-999
1821-999
1822-999
1823-999
1824-999
1825-999
1826-999
1827-999
1828-999
1829-999
1830-999
1831-999
1832-999
1833-999
1834-999
1835-999
1836-999
1837-999
1838-999
1839-999
1840-999
1841-999
1842-999
1843-999
1844-999
1845-999
1846-999
1847-999
1848-999
1849-999
1850-999
1851-999
1852-999
1853-999
1854-999
1855-999
1856-999
1857-999
1858-999
1859-999
1860-999
1861-999
1862-999
1863-999
1864-999
1865-999
1866-999
1867-999
1868-999
1869-999
1870-999
1871-999
1872-999
1873-999
1874-999
1875-999
1876-999
1877-999
1878-999
1879-999
1880-999
1881-999
1882-999
1883-999
1884-999
1885-999
1886-999
1887-999
1888-999
1889-999
1890-999
1891-999
1892-999
1893-999
1894-999
1895-999
1896-999
1897-999
1898-999
1899-999
1900-999
1901-999
1902-999
1903-999
1904-999
1905-999
1906-999
1907-999
1908-999
1909-999
1910-999
1911-999
1912-999
1913-999
1914-999
1915-999
1916-999
1917-999
1918-999
1919-999
1920-999
1921-999
1922-999
1923-999
1924-999
1925-999
1926-999
1927-999
1928-999
1929-999
1930-999
1931-999
1932-999
1933-999
1934-999
1935-999
1936-999
1937-999
1938-999
1939-999
1940-999
1941-999
1942-999
1943-999
1944-999
1945-999
1946-999
1947-999
1948-999
1949-999
1950-999
1951-999
1952-999
1953-999
1954-999
1955-999
1956-999
1957-999
1958-999
1959-999
1960-999
1961-999
1962-999
1963-999
1964-999
1965-999
1966-999
1967-999
1968-999
1969-999
1970-999
1971-999
1972-999
1973-999
1974-999
1975-999
1976-999
1977-999
1978-999
1979-999
1980-999
1981-999
1982-999
1983-999
1984-999
1985-999
1986-999
1987-999
1988-999
1989-999
1990-999
1991-999
1992-999
1993-999
1994-999
1995-999
1996-999
1997-999
1998-999
1999-999
2000-999
2001-999
2002-999
2003-999
2004-999
2005-999
2006-999
2007-999
2008-999
2009-999
2010-999
2011-999
2012-999
2013-999
2014-999
2015-999
2016-999
2017-999
2018-999
2019-999
2020-999
2021-999
2022-999
2023-999
2024-999
2025-999
2026-999
2027-999
2028-999
2029-999
2030-999
2031-999
2032-999
2033-999
2034-999
2035-999
2036-999
2037-999
2038-999
2039-999
2040-999
2041-999
2042-999
2043-999
2044-999
2045-999
2046-999
2047-999
2048-999
2049-999
2050-999
2051-999
2052-999
2053-999
2054-999
2055-999
2056-999
2057-999
2058-999
2059-999
2060-999
2061-999
2062-999
2063-999
2064-999
2065-999
2066-999
2067-999
2068-999
2069-999
2070-999
2071-999
2072-999
2073-999
2074-999
2075-999
2076-999
2077-999
2078-999
2079-999
2080-999
2081-999
2082-999
2083-999
2084-999
2085-999
2086-999
2087-999
2088-999
2089-999
2090-999
2091-999
2092-999
2093-999
2094-999
2095-999
2096-999
2097-999
2098-999
2099-999
2100-999
2101-999
2102-999
2103-999
2104-999
2105-999
2106-999
2107-999
2108-999
2109-999
2110-999
2111-999
2112-999
2113-999
2114-999
2115-999
2116-999
2117-999
2118-999
2119-999
2120-999
2121-999
2122-999
2123-999
2124-999
2125-999
2126-999
2127-999
2128-999
2129-999
2130-999
2131-999
2132-999
2133-999
2134-999
2135-999
2136-999
2137-999
2138-999
2139-999
2140-999
2141-999
2142-999
2143-999
2144-999
2145-999
2146-999
2147-999
2148-999
2149-999
2150-999
2151-999
2152-999
2153-999
2154-999
2155-999
2156-999
2157-999
2158-999
2159-999
2160-999
2161-999
2162-999
2163-999
2164-999
2165-999
2166-999
2167-999
2168-999
2169-999
2170-999
2171-999
2172-999
2173-999
2174-999
2175-999
2176-999
2177-999
2178-999
2179-999
2180-999
2181-999
2182-999
2183-999
2184-999
2185-999
2186-999
2187-999
2188-999
2189-999
2190-999
2191-999
2192-999
2193-999
2194-999
2195-999
2196-999
2197-999
2198-999
2199-999
2200-999
2201-999
2202-999
2203-999
2204-999
2205-999
2206-999
2207-999
2208-999
2209-999
2210-999
2211-999
2212-999
2213-999
2214-999
2215-999
2216-999
2217-999
2218-999
2219-999
2220-999
2221-999
2222-999
2223-999
2224-999
2225-999
2226-999
2227-999
2228-999
2229-999
2230-999
2231-999
2232-999
2233-999
2234-999
2235-999
2236-999
2237-999
2238-999
2239-999
2240-999
2241-999
2242-999
2243-999
2244-999
2245-999
2246-999
2247-999
2248-999
2249-999
2250-999
2251-999
2252-999
2253-999
2254-999
2255-999
2256-999
2257-999
2258-999
2259-999
2260-999
2261-999
2262-999
2263-999
2264-999
2265-999
2266-999
2267-999
2268-999
2269-999
2270-999
2271-999
2272-999
2273-999
2274-999
2275-999
2276-999
2277-999
2278-999
2279-999
2280-999
2281-999
2282-999
2283-999
2284-999
2285-999
2286-999
2287-999
2288-999
2289-999
2290-999
2291-999
2292-999
2293-999
2294-999
2295-999
2296-999
2297-999
2298-999
2299-999
2300-999
2301-999
2302-999
2303-999
2304-999
2305-999
2306-999
2307-999
2308-999
2309-999
2310-999
2311-999
2312-999
2313-999
2314-999
2315-999
2316-999
2317-999
2318-999
2319-999
2320-999
2321-999
2322-999
2323-999
2324-999
2325-999
2326-999
2327-999
2328-999
2329-999
2330-999
2331-999
2332-999
2333-999
2334-999
2335-999
2336-999
2337-999
2338-999
2339-999
2340-999
2341-999
2342-999
2343-999
2344-999
2345-999
2346-999
2347-999
2348-999
2349-999
2350-999
2351-999
2352-999
2353-999
2354-999
2355-999
2356-999
2357-999
2358-999
2359-999
2360-999
2361-999
2362-999
2363-999
2364-999
2365-999
2366-999
2367-999
2368-999
2369-999
2370-999
2371-999
2372-999
2373-999
2374-999
2375-999
2376-999
2377-999
2378-999
2379-999
2380-999
2381-999
2382-999
2383-999
2384-999
2385-999
2386-999
2387-999
2388-999
2389-999
2390-999
2391-999
2392-999
2393-999
2394-999
2395-999
2396-999
2397-999
2398-999
2399-999
2400-999
2401-999
2402-999
2403-999
2404-999
2405-999
2406-999
2407-999
2408-999
2409-999
2410-999
2411-999
2412-999
2413-999
2414-999
2415-999
2416-999
2417-999
2418-999
2419-999
2420-999
2421-999
2422-999
2423-999
2424-999
2425-999
2426-999
2427-999
2428-999
2429-999
2430-999
2431-999
2432-999
2433-999
2434-999
2435-999
2436-999
2437-999
2438-999
2439-999
2440-999
2441-999
2442-999
2443-999
2444-999
2445-999
2446-999
2447-999
2448-999
2449-999
2450-999
2451-999
2452-999
2453-999
2454-999
2455-999
2456-999
2457-999
2458-999
2459-999
2460-999
2461-999
2462-999
2463-999
2464-999
2465-999
2466-999
2467-999
2468-999
2469-999
2470-999
2471-999
2472-999
2473-999
2474-999
2475-999
2476-999
2477-999
2478-999
2479-999
2480-999
2481-999
2482-999
2483-999
2484-999
2485-999
2486-999
2487-999
2488-999
2489-999
2490-999
2491-999
2492-999
2493-999
2494-999
2495-999
2496-999
2497-999
2498-999
2499-999
2500-999
2501-999
2502-999
2503-999
2504-999
2505-999
2506-999
2507-999
2508-999
2509-999
2510-999
2511-999
2512-999
2513-999
2514-999
2515-999
2516-999
2517-999
2518-999
2519-999
2520-999
2521-999
2522-999
2523-999
2524-999
2525-999
2526-999
2527-999
2528-999
2529-999
2530-999
2531-999
2532-999
2533-999
2534-999
2535-999
2536-999
2537-999
2538-999
2539-999
2540-999
2541-999
2542-999
2543-999
2544-999
2545-999
2546-999
2547-999
2548-999
2549-999
2550-999
2551-999
2552-999
2553-999
2554-999
2555-999
2556-999
2557-999
2558-999
2559-999
2560-999
2561-999
2562-999
2563-999
2564-999
2565-999
2566-999
2567-999
2568-999
2569-999
2570-999
2571-999
2572-999
2573-999
2574-999
2575-999
2576-999
2577-999
2578-999
2579-999
2580-999
2581-999
2582-999
2583-999
2584-999
2585-999
2586-999
2587-999
2588-999
2589-999
2590-999
2591-999
2592-999
2593-999
2594-999
2595-999
2596-999
2597-999
2598-999
2599-999
2600-999
2601-999
2602-999
2603-999
2604-999
2605-999
2606-999
2607-999
2608-999
2609-999
2610-999
2611-999
2612-999
2613-999
2614-999
2615-999
2616-999
2617-999
2618-999
2619-999
2620-999
2621-999
2622-999
2623-999
2624-999
2625-999
2626-999
2627-999
2628-999
2629-999
2630-999
2631-999
2632-999
2633-999
2634-999
2635-999
2636-999
2637-999
2638-999
2639-999
2640-999
2641-999
2642-999
2643-999
2644-999
2645-999
2646-999
2647-999
2648-999
2649-999
2650-999
2651-999
2652-999
2653-999
2654-999
2655-999
2656-999
2657-999
2658-999
2659-999
2660-999
2661-999
2662-999
2663-999
2664-999
2665-999
2666-999
2667-999
2668-999
2669-999
2670-999
2671-999
2672-999
2673-999
2674-999
2675-999
2676-999
2677-999
2678-999
2679-999
2680-999
2681-999
2682-999
2683-999
2684-999
2685-999
2686-999
2687-999
2688-999
2689-999
2690-999
2691-999
2692-999
2693-999
2694-999
2695-999
2696-999
2697-999
2698-999
2699-999
2700-999
2701-999
2702-999
2703-999
2704-999
2705-999
2706-999
2707-999
2708-999
2709-999
2710-999
2711-999
2712-999
2713-999
2714-999
2715-999
2716-999
2717-999
2718-999
2719-999
2720-999
2721-999
2722-999
2723-999
2724-999
2725-999
2726-999
2727-999
2728-999
2729-999
2730-999
2731-999
2732-999
2733-999
2734-999
2735-999
2736-999
2737-999
2738-999
2739-999
2740-999
2741-999
2742-999
2743-999
2744-999
2745-999
2746-999
2747-999
2748-999
2749-999
2750-999
2751-999
2752-999
2753-999
2754-999
2755-999
2756-999
2757-999
2758-999
2759-999
2760-999
2761-999
2762-999
2763-999
2764-999
2765-999
2766-999
2767-999
2768-999
2769-999
2770-999
2771-999
2772-999
2773-999
2774-999
2775-999
2776-999
2777-999
2778-999
2779-999
2780-999
2781-999
2782-999
2783-999
2784-999
2785-999
2786-999
2787-999
2788-999
2789-999
2790-999
2791-999
2792-999
2793-999
2794-999
2795-999
2796-999
2797-999
2798-999
2799-999
2800-999
2801-999
2802-999
2803-999
2804-999
2805-999
2806-999
2807-999
2808-999
2809-999
2810-999
2811-999
2812-999
2813-999
2814-999
2815-999
2816-999
2817-999
2818-999
2819-999
2820-999
2821-999
2822-999
2823-999
2824-999
2825-999
2826-999
2827-999
2828-999
2829-999
2830-999
2831-999
2832-999
2833-999
2834-999
2835-999
2836-999
2837-999
2838-999
2839-999
2840-999
2841-999
2842-999
2843-999
2844-999
2845-999
2846-999
2847-999
2848-999
2849-999
2850-999
2851-999
2852-999
2853-999
2854-999
2855-999
2856-999
2857-999
2858-999
2859-999
2860-999
2861-999
2862-999
2863-999
2864-999
2865-999
2866-999
2867-999
2868-999
2869-999
2870-999
2871-999
2872-999
2873-999
2874-999
2875-999
2876-999
2877-999
2878-999
2879-999
2880-999
2881-999
2882-999
2883-999
2884-999
2885-999
2886-999
2887-999
2888-999
2889-999
2890-999
2891-999
2892-999
2893-999
2894-999
2895-999
2896-999
2897-999
2898-999
2899-999
2900-999
2901-999
2902-999
2903-999
2904-999
2905-999
2906-999
2907-999
2908-999
2909-999
2910-999
2911-999
2912-999
2913-999
2914-999
2915-999
2916-999
2917-999
2918-999
2919-999
2920-999
2921-999
2922-999
2923-999
2924-999
2925-999
2926-999
2927-999
2928-999
2929-999
2930-999
2931-999
2932-999
2933-999
2934-999
2935-999
2936-999
2937-999
2938-999
2939-999
2940-999
2941-999
2942-999
2943-999
2944-999
2945-999
2946-999
2947-999
2948-999
2949-999
2950-999
2951-999
2952-999
2953-999
2954-999
2955-999
2956-999
2957-999
2958-999
2959-999
2960-999
2961-999
2962-999
2963-999
2964-999
2965-999
2966-999
2967-999
2968-999
2969-999
2970-999
2971-999
2972-999
2973-999
2974-999
2975-999
2976-999
2977-999
2978-999
2979-999
2980-999
2981-999
2982-999
2983-999
2984-999
2985-999
2986-999
2987-999
2988-999
2989-999
2990-999
2991-999
2992-999
2993-999
2994-999
2995-999
2996-999
2997-999
2998-999
2999-999
3000-999
3001-999
3002-999
3003-999
3004-999
3005-999
3006-999
3007-999
3008-999
3009-999
3010-999
3011-999
3012-999
3013-999
3014-999
3015-999
3016-999
3017-999
3018-999
3019-999
3020-999
3021-999
3022-999
3023-999
3024-999
3025-999
3026-999
3027-999
3028-999
3029-999
3030-999
3031-999
3032-999
3033-999
3034-999
3035-999
3036-999
3037-999
3038-999
3039-999
3040-999
3041-999
3042-999
3043-999
3044-999
3045-999
3046-999
3047-999
3048-999
3049-999
3050-999
3051-999
3052-999
3053-999
3054-999
3055-999
3056-999
3057-999
3058-999
3059-999
3060-999
3061-999
3062-999
3063-999
3064-999
3065-999
3066-999
3067-999
3068-999
3069-999
3070-999
3071-999
3072-999
3073-999
1-999a typea comment-999.0
2-999a typea comment-999.0
3-999a typea comment-999.0
4-999a typea comment-999.0
5-999a typea comment-999.0
6-999a typea comment-999.0
7-999a typea comment-999.0
8-999a typea comment-999.0
9-999a typea comment-999.0
10-999a typea comment-999.0
11-999a typea comment-999.0
12-999a typea comment-999.0
13-999a typea comment-999.0
14-999a typea comment-999.0
15-999a typea comment-999.0
16-999a typea comment-999.0
17-999a typea comment-999.0
18-999a typea comment-999.0
19-999a typea comment-999.0
20-999a typea comment-999.0
21-999a typea comment-999.0
22-999a typea comment-999.0
23-999a typea comment-999.0
24-999a typea comment-999.0
25-999a typea comment-999.0
26-999a typea comment-999.0
27-999a typea comment-999.0
28-999a typea comment-999.0
29-999a typea comment-999.0
30-999a typea comment-999.0
31-999a typea comment-999.0
32-999a typea comment-999.0
33-999a typea comment-999.0
34-999a typea comment-999.0
35-999a typea comment-999.0
36-999a typea comment-999.0
37-999a typea comment-999.0
38-999a typea comment-999.0
39-999a typea comment-999.0
40-999a typea comment-999.0
41-999a typea comment-999.0
42-999a typea comment-999.0
43-999a typea comment-999.0
44-999a typea comment-999.0
45-999a typea comment-999.0
46-999a typea comment-999.0
47-999a typea comment-999.0
48-999a typea comment-999.0
49-999a typea comment-999.0
50-999a typea comment-999.0
51-999a typea comment-999.0
52-999a typea comment-999.0
53-999a typea comment-999.0
54-999a typea comment-999.0
55-999a typea comment-999.0
56-999a typea comment-999.0
57-999a typea comment-999.0
58-999a typea comment-999.0
59-999a typea comment-999.0
60-999a typea comment-999.0
61-999a typea comment-999.0
62-999a typea comment-999.0
63-999a typea comment-999.0
64-999a typea comment-999.0
65-999a typea comment-999.0
66-999a typea comment-999.0
67-999a typea comment-999.0
68-999a typea comment-999.0
69-999a typea comment-999.0
70-999a typea comment-999.0
71-999a typea comment-999.0
72-999a typea comment-999.0
73-999a typea comment-999.0
74-999a typea comment-999.0
75-999a typea comment-999.0
76-999a typea comment-999.0
77-999a typea comment-999.0
78-999a typea comment-999.0
79-999a typea comment-999.0
80-999a typea comment-999.0
81-999a typea comment-999.0
82-999a typea comment-999.0
83-999a typea comment-999.0
84-999a typea comment-999.0
85-999a typea comment-999.0
86-999a typea comment-999.0
87-999a typea comment-999.0
88-999a typea comment-999.0
89-999a typea comment-999.0
90-999a typea comment-999.0
91-999a typea comment-999.0
92-999a typea comment-999.0
93-999a typea comment-999.0
94-999a typea comment-999.0
95-999a typea comment-999.0
96-999a typea comment-999.0
97-999a typea comment-999.0
98-999a typea comment-999.0
99-999a typea comment-999.0
100-999a typea comment-999.0
101-999a typea comment-999.0
102-999a typea comment-999.0
103-999a typea comment-999.0
104-999a typea comment-999.0
105-999a typea comment-999.0
106-999a typea comment-999.0
107-999a typea comment-999.0
108-999a typea comment-999.0
109-999a typea comment-999.0
110-999a typea comment-999.0
111-999a typea comment-999.0
112-999a typea comment-999.0
113-999a typea comment-999.0
114-999a typea comment-999.0
115-999a typea comment-999.0
116-999a typea comment-999.0
117-999a typea comment-999.0
118-999a typea comment-999.0
119-999a typea comment-999.0
120-999a typea comment-999.0
121-999a typea comment-999.0
122-999a typea comment-999.0
123-999a typea comment-999.0
124-999a typea comment-999.0
125-999a typea comment-999.0
126-999a typea comment-999.0
127-999a typea comment-999.0
128-999a typea comment-999.0
129-999a typea comment-999.0
130-999a typea comment-999.0
131-999a typea comment-999.0
132-999a typea comment-999.0
133-999a typea comment-999.0
134-999a typea comment-999.0
135-999a typea comment-999.0
136-999a typea comment-999.0
137-999a typea comment-999.0
138-999a typea comment-999.0
139-999a typea comment-999.0
140-999a typea comment-999.0
141-999a typea comment-999.0
142-999a typea comment-999.0
143-999a typea comment-999.0
144-999a typea comment-999.0
145-999a typea comment-999.0
146-999a typea comment-999.0
147-999a typea comment-999.0
148-999a typea comment-999.0
149-999a typea comment-999.0
150-999a typea comment-999.0
151-999a typea comment-999.0
152-999a typea comment-999.0
153-999a typea comment-999.0
154-999a typea comment-999.0
155-999a typea comment-999.0
156-999a typea comment-999.0
157-999a typea comment-999.0
158-999a typea comment-999.0
159-999a typea comment-999.0
160-999a typea comment-999.0
161-999a typea comment-999.0
162-999a typea comment-999.0
163-999a typea comment-999.0
164-999a typea comment-999.0
165-999a typea comment-999.0
166-999a typea comment-999.0
167-999a typea comment-999.0
168-999a typea comment-999.0
169-999a typea comment-999.0
170-999a typea comment-999.0
171-999a typea comment-999.0
172-999a typea comment-999.0
173-999a typea comment-999.0
174-999a typea comment-999.0
175-999a typea comment-999.0
176-999a typea comment-999.0
177-999a typea comment-999.0
178-999a typea comment-999.0
179-999a typea comment-999.0
180-999a typea comment-999.0
181-999a typea comment-999.0
182-999a typea comment-999.0
183-999a typea comment-999.0
184-999a typea comment-999.0
185-999a typea comment-999.0
186-999a typea comment-999.0
187-999a typea comment-999.0
188-999a typea comment-999.0
189-999a typea comment-999.0
190-999a typea comment-999.0
191-999a typea comment-999.0
192-999a typea comment-999.0
193-999a typea comment-999.0
194-999a typea comment-999.0
195-999a typea comment-999.0
196-999a typea comment-999.0
197-999a typea comment-999.0
198-999a typea comment-999.0
199-999a typea comment-999.0
200-999a typea comment-999.0
201-999a typea comment-999.0
202-999a typea comment-999.0
203-999a typea comment-999.0
204-999a typea comment-999.0
205-999a typea comment-999.0
206-999a typea comment-999.0
207-999a typea comment-999.0
208-999a typea comment-999.0
209-999a typea comment-999.0
210-999a typea comment-999.0
211-999a typea comment-999.0
212-999a typea comment-999.0
213-999a typea comment-999.0
214-999a typea comment-999.0
215-999a typea comment-999.0
216-999a typea comment-999.0
217-999a typea comment-999.0
218-999a typea comment-999.0
219-999a typea comment-999.0
220-999a typea comment-999.0
221-999a typea comment-999.0
222-999a typea comment-999.0
223-999a typea comment-999.0
224-999a typea comment-999.0
225-999a typea comment-999.0
226-999a typea comment-999.0
227-999a typea comment-999.0
228-999a typea comment-999.0
229-999a typea comment-999.0
230-999a typea comment-999.0
231-999a typea comment-999.0
232-999a typea comment-999.0
233-999a typea comment-999.0
234-999a typea comment-999.0
235-999a typea comment-999.0
236-999a typea comment-999.0
237-999a typea comment-999.0
238-999a typea comment-999.0
239-999a typea comment-999.0
240-999a typea comment-999.0
241-999a typea comment-999.0
242-999a typea comment-999.0
243-999a typea comment-999.0
244-999a typea comment-999.0
245-999a typea comment-999.0
246-999a typea comment-999.0
247-999a typea comment-999.0
248-999a typea comment-999.0
249-999a typea comment-999.0
250-999a typea comment-999.0
251-999a typea comment-999.0
252-999a typea comment-999.0
253-999a typea comment-999.0
254-999a typea comment-999.0
255-999a typea comment-999.0
256-999a typea comment-999.0
257-999a typea comment-999.0
258-999a typea comment-999.0
259-999a typea comment-999.0
260-999a typea comment-999.0
261-999a typea comment-999.0
262-999a typea comment-999.0
263-999a typea comment-999.0
264-999a typea comment-999.0
265-999a typea comment-999.0
266-999a typea comment-999.0
267-999a typea comment-999.0
268-999a typea comment-999.0
269-999a typea comment-999.0
270-999a typea comment-999.0
271-999a typea comment-999.0
272-999a typea comment-999.0
273-999a typea comment-999.0
274-999a typea comment-999.0
275-999a typea comment-999.0
276-999a typea comment-999.0
277-999a typea comment-999.0
278-999a typea comment-999.0
279-999a typea comment-999.0
280-999a typea comment-999.0
281-999a typea comment-999.0
282-999a typea comment-999.0
283-999a typea comment-999.0
284-999a typea comment-999.0
285-999a typea comment-999.0
286-999a typea comment-999.0
287-999a typea comment-999.0
288-999a typea comment-999.0
289-999a typea comment-999.0
290-999a typea comment-999.0
291-999a typea comment-999.0
292-999a typea comment-999.0
293-999a typea comment-999.0
294-999a typea comment-999.0
295-999a typea comment-999.0
296-999a typea comment-999.0
297-999a typea comment-999.0
298-999a typea comment-999.0
299-999a typea comment-999.0
300-999a typea comment-999.0
301-999a typea comment-999.0
302-999a typea comment-999.0
303-999a typea comment-999.0
304-999a typea comment-999.0
305-999a typea comment-999.0
306-999a typea comment-999.0
307-999a typea comment-999.0
308-999a typea comment-999.0
309-999a typea comment-999.0
310-999a typea comment-999.0
311-999a typea comment-999.0
312-999a typea comment-999.0
313-999a typea comment-999.0
314-999a typea comment-999.0
315-999a typea comment-999.0
316-999a typea comment-999.0
317-999a typea comment-999.0
318-999a typea comment-999.0
319-999a typea comment-999.0
320-999a typea comment-999.0
321-999a typea comment-999.0
322-999a typea comment-999.0
323-999a typea comment-999.0
324-999a typea comment-999.0
325-999a typea comment-999.0
326-999a typea comment-999.0
327-999a typea comment-999.0
328-999a typea comment-999.0
329-999a typea comment-999.0
330-999a typea comment-999.0
331-999a typea comment-999.0
332-999a typea comment-999.0
333-999a typea comment-999.0
334-999a typea comment-999.0
335-999a typea comment-999.0
336-999a typea comment-999.0
337-999a typea comment-999.0
338-999a typea comment-999.0
339-999a typea comment-999.0
340-999a typea comment-999.0
341-999a typea comment-999.0
342-999a typea comment-999.0
343-999a typea comment-999.0
344-999a typea comment-999.0
345-999a typea comment-999.0
346-999a typea comment-999.0
347-999a typea comment-999.0
348-999a typea comment-999.0
349-999a typea comment-999.0
350-999a typea comment-999.0
351-999a typea comment-999.0
352-999a typea comment-999.0
353-999a typea comment-999.0
354-999a typea comment-999.0
355-999a typea comment-999.0
356-999a typea comment-999.0
357-999a typea comment-999.0
358-999a typea comment-999.0
359-999a typea comment-999.0
360-999a typea comment-999.0
361-999a typea comment-999.0
362-999a typea comment-999.0
363-999a typea comment-999.0
364-999a typea comment-999.0
365-999a typea comment-999.0
366-999a typea comment-999.0
367-999a typea comment-999.0
368-999a typea comment-999.0
369-999a typea comment-999.0
370-999a typea comment-999.0
371-999a typea comment-999.0
372-999a typea comment-999.0
373-999a typea comment-999.0
374-999a typea comment-999.0
375-999a typea comment-999.0
376-999a typea comment-999.0
377-999a typea comment-999.0
378-999a typea comment-999.0
379-999a typea comment-999.0
380-999a typea comment-999.0
381-999a typea comment-999.0
382-999a typea comment-999.0
383-999a typea comment-999.0
384-999a typea comment-999.0
385-999a typea comment-999.0
386-999a typea comment-999.0
387-999a typea comment-999.0
388-999a typea comment-999.0
389-999a typea comment-999.0
390-999a typea comment-999.0
391-999a typea comment-999.0
392-999a typea comment-999.0
393-999a typea comment-999.0
394-999a typea comment-999.0
395-999a typea comment-999.0
396-999a typea comment-999.0
397-999a typea comment-999.0
398-999a typea comment-999.0
399-999a typea comment-999.0
400-999a typea comment-999.0
401-999a typea comment-999.0
402-999a typea comment-999.0
403-999a typea comment-999.0
404-999a typea comment-999.0
405-999a typea comment-999.0
406-999a typea comment-999.0
407-999a typea comment-999.0
408-999a typea comment-999.0
409-999a typea comment-999.0
410-999a typea comment-999.0
411-999a typea comment-999.0
412-999a typea comment-999.0
413-999a typea comment-999.0
414-999a typea comment-999.0
415-999a typea comment-999.0
416-999a typea comment-999.0
417-999a typea comment-999.0
418-999a typea comment-999.0
419-999a typea comment-999.0
420-999a typea comment-999.0
421-999a typea comment-999.0
422-999a typea comment-999.0
423-999a typea comment-999.0
424-999a typea comment-999.0
425-999a typea comment-999.0
426-999a typea comment-999.0
427-999a typea comment-999.0
428-999a typea comment-999.0
429-999a typea comment-999.0
430-999a typea comment-999.0
431-999a typea comment-999.0
432-999a typea comment-999.0
433-999a typea comment-999.0
434-999a typea comment-999.0
435-999a typea comment-999.0
436-999a typea comment-999.0
437-999a typea comment-999.0
438-999a typea comment-999.0
439-999a typea comment-999.0
440-999a typea comment-999.0
441-999a typea comment-999.0
442-999a typea comment-999.0
443-999a typea comment-999.0
444-999a typea comment-999.0
445-999a typea comment-999.0
446-999a typea comment-999.0
447-999a typea comment-999.0
448-999a typea comment-999.0
449-999a typea comment-999.0
450-999a typea comment-999.0
451-999a typea comment-999.0
452-999a typea comment-999.0
453-999a typea comment-999.0
454-999a typea comment-999.0
455-999a typea comment-999.0
456-999a typea comment-999.0
457-999a typea comment-999.0
458-999a typea comment-999.0
459-999a typea comment-999.0
460-999a typea comment-999.0
461-999a typea comment-999.0
462-999a typea comment-999.0
463-999a typea comment-999.0
464-999a typea comment-999.0
465-999a typea comment-999.0
466-999a typea comment-999.0
467-999a typea comment-999.0
468-999a typea comment-999.0
469-999a typea comment-999.0
470-999a typea comment-999.0
471-999a typea comment-999.0
472-999a typea comment-999.0
473-999a typea comment-999.0
474-999a typea comment-999.0
475-999a typea comment-999.0
476-999a typea comment-999.0
477-999a typea comment-999.0
478-999a typea comment-999.0
479-999a typea comment-999.0
480-999a typea comment-999.0
481-999a typea comment-999.0
482-999a typea comment-999.0
483-999a typea comment-999.0
484-999a typea comment-999.0
485-999a typea comment-999.0
486-999a typea comment-999.0
487-999a typea comment-999.0
488-999a typea comment-999.0
489-999a typea comment-999.0
490-999a typea comment-999.0
491-999a typea comment-999.0
492-999a typea comment-999.0
493-999a typea comment-999.0
494-999a typea comment-999.0
495-999a typea comment-999.0
496-999a typea comment-999.0
497-999a typea comment-999.0
498-999a typea comment-999.0
499-999a typea comment-999.0
500-999a typea comment-999.0
501-999a typea comment-999.0
502-999a typea comment-999.0
503-999a typea comment-999.0
504-999a typea comment-999.0
505-999a typea comment-999.0
506-999a typea comment-999.0
507-999a typea comment-999.0
508-999a typea comment-999.0
509-999a typea comment-999.0
510-999a typea comment-999.0
511-999a typea comment-999.0
512-999a typea comment-999.0
513-999a typea comment-999.0
514-999a typea comment-999.0
515-999a typea comment-999.0
516-999a typea comment-999.0
517-999a typea comment-999.0
518-999a typea comment-999.0
519-999a typea comment-999.0
520-999a typea comment-999.0
521-999a typea comment-999.0
522-999a typea comment-999.0
523-999a typea comment-999.0
524-999a typea comment-999.0
525-999a typea comment-999.0
526-999a typea comment-999.0
527-999a typea comment-999.0
528-999a typea comment-999.0
529-999a typea comment-999.0
530-999a typea comment-999.0
531-999a typea comment-999.0
532-999a typea comment-999.0
533-999a typea comment-999.0
534-999a typea comment-999.0
535-999a typea comment-999.0
536-999a typea comment-999.0
537-999a typea comment-999.0
538-999a typea comment-999.0
539-999a typea comment-999.0
540-999a typea comment-999.0
541-999a typea comment-999.0
542-999a typea comment-999.0
543-999a typea comment-999.0
544-999a typea comment-999.0
545-999a typea comment-999.0
546-999a typea comment-999.0
547-999a typea comment-999.0
548-999a typea comment-999.0
549-999a typea comment-999.0
550-999a typea comment-999.0
551-999a typea comment-999.0
552-999a typea comment-999.0
553-999a typea comment-999.0
554-999a typea comment-999.0
555-999a typea comment-999.0
556-999a typea comment-999.0
557-999a typea comment-999.0
558-999a typea comment-999.0
559-999a typea comment-999.0
560-999a typea comment-999.0
561-999a typea comment-999.0
562-999a typea comment-999.0
563-999a typea comment-999.0
564-999a typea comment-999.0
565-999a typea comment-999.0
566-999a typea comment-999.0
567-999a typea comment-999.0
568-999a typea comment-999.0
569-999a typea comment-999.0
570-999a typea comment-999.0
571-999a typea comment-999.0
572-999a typea comment-999.0
573-999a typea comment-999.0
574-999a typea comment-999.0
575-999a typea comment-999.0
576-999a typea comment-999.0
577-999a typea comment-999.0
578-999a typea comment-999.0
579-999a typea comment-999.0
580-999a typea comment-999.0
581-999a typea comment-999.0
582-999a typea comment-999.0
583-999a typea comment-999.0
584-999a typea comment-999.0
585-999a typea comment-999.0
586-999a typea comment-999.0
587-999a typea comment-999.0
588-999a typea comment-999.0
589-999a typea comment-999.0
590-999a typea comment-999.0
591-999a typea comment-999.0
592-999a typea comment-999.0
593-999a typea comment-999.0
594-999a typea comment-999.0
595-999a typea comment-999.0
596-999a typea comment-999.0
597-999a typea comment-999.0
598-999a typea comment-999.0
599-999a typea comment-999.0
600-999a typea comment-999.0
601-999a typea comment-999.0
602-999a typea comment-999.0
603-999a typea comment-999.0
604-999a typea comment-999.0
605-999a typea comment-999.0
606-999a typea comment-999.0
607-999a typea comment-999.0
608-999a typea comment-999.0
609-999a typea comment-999.0
610-999a typea comment-999.0
611-999a typea comment-999.0
612-999a typea comment-999.0
613-999a typea comment-999.0
614-999a typea comment-999.0
615-999a typea comment-999.0
616-999a typea comment-999.0
617-999a typea comment-999.0
618-999a typea comment-999.0
619-999a typea comment-999.0
620-999a typea comment-999.0
621-999a typea comment-999.0
622-999a typea comment-999.0
623-999a typea comment-999.0
624-999a typea comment-999.0
625-999a typea comment-999.0
626-999a typea comment-999.0
627-999a typea comment-999.0
628-999a typea comment-999.0
629-999a typea comment-999.0
630-999a typea comment-999.0
631-999a typea comment-999.0
632-999a typea comment-999.0
633-999a typea comment-999.0
634-999a typea comment-999.0
635-999a typea comment-999.0
636-999a typea comment-999.0
637-999a typea comment-999.0
638-999a typea comment-999.0
639-999a typea comment-999.0
640-999a typea comment-999.0
641-999a typea comment-999.0
642-999a typea comment-999.0
643-999a typea comment-999.0
644-999a typea comment-999.0
645-999a typea comment-999.0
646-999a typea comment-999.0
647-999a typea comment-999.0
648-999a typea comment-999.0
649-999a typea comment-999.0
650-999a typea comment-999.0
651-999a typea comment-999.0
652-999a typea comment-999.0
653-999a typea comment-999.0
654-999a typea comment-999.0
655-999a typea comment-999.0
656-999a typea comment-999.0
657-999a typea comment-999.0
658-999a typea comment-999.0
659-999a typea comment-999.0
660-999a typea comment-999.0
661-999a typea comment-999.0
662-999a typea comment-999.0
663-999a typea comment-999.0
664-999a typea comment-999.0
665-999a typea comment-999.0
666-999a typea comment-999.0
667-999a typea comment-999.0
668-999a typea comment-999.0
669-999a typea comment-999.0
670-999a typea comment-999.0
671-999a typea comment-999.0
672-999a typea comment-999.0
673-999a typea comment-999.0
674-999a typea comment-999.0
675-999a typea comment-999.0
676-999a typea comment-999.0
677-999a typea comment-999.0
678-999a typea comment-999.0
679-999a typea comment-999.0
680-999a typea comment-999.0
681-999a typea comment-999.0
682-999a typea comment-999.0
683-999a typea comment-999.0
684-999a typea comment-999.0
685-999a typea comment-999.0
686-999a typea comment-999.0
687-999a typea comment-999.0
688-999a typea comment-999.0
689-999a typea comment-999.0
690-999a typea comment-999.0
691-999a typea comment-999.0
692-999a typea comment-999.0
693-999a typea comment-999.0
694-999a typea comment-999.0
695-999a typea comment-999.0
696-999a typea comment-999.0
697-999a typea comment-999.0
698-999a typea comment-999.0
699-999a typea comment-999.0
700-999a typea comment-999.0
701-999a typea comment-999.0
702-999a typea comment-999.0
703-999a typea comment-999.0
704-999a typea comment-999.0
705-999a typea comment-999.0
706-999a typea comment-999.0
707-999a typea comment-999.0
708-999a typea comment-999.0
709-999a typea comment-999.0
710-999a typea comment-999.0
711-999a typea comment-999.0
712-999a typea comment-999.0
713-999a typea comment-999.0
714-999a typea comment-999.0
715-999a typea comment-999.0
716-999a typea comment-999.0
717-999a typea comment-999.0
718-999a typea comment-999.0
719-999a typea comment-999.0
720-999a typea comment-999.0
721-999a typea comment-999.0
722-999a typea comment-999.0
723-999a typea comment-999.0
724-999a typea comment-999.0
725-999a typea comment-999.0
726-999a typea comment-999.0
727-999a typea comment-999.0
728-999a typea comment-999.0
729-999a typea comment-999.0
730-999a typea comment-999.0
731-999a typea comment-999.0
732-999a typea comment-999.0
733-999a typea comment-999.0
734-999a typea comment-999.0
735-999a typea comment-999.0
736-999a typea comment-999.0
737-999a typea comment-999.0
738-999a typea comment-999.0
739-999a typea comment-999.0
740-999a typea comment-999.0
741-999a typea comment-999.0
742-999a typea comment-999.0
743-999a typea comment-999.0
744-999a typea comment-999.0
745-999a typea comment-999.0
746-999a typea comment-999.0
747-999a typea comment-999.0
748-999a typea comment-999.0
749-999a typea comment-999.0
750-999a typea comment-999.0
751-999a typea comment-999.0
752-999a typea comment-999.0
753-999a typea comment-999.0
754-999a typea comment-999.0
755-999a typea comment-999.0
756-999a typea comment-999.0
757-999a typea comment-999.0
758-999a typea comment-999.0
759-999a typea comment-999.0
760-999a typea comment-999.0
761-999a typea comment-999.0
762-999a typea comment-999.0
763-999a typea comment-999.0
764-999a typea comment-999.0
765-999a typea comment-999.0
766-999a typea comment-999.0
767-999a typea comment-999.0
768-999a typea comment-999.0
769-999a typea comment-999.0
770-999a typea comment-999.0
771-999a typea comment-999.0
772-999a typea comment-999.0
773-999a typea comment-999.0
774-999a typea comment-999.0
775-999a typea comment-999.0
776-999a typea comment-999.0
777-999a typea comment-999.0
778-999a typea comment-999.0
779-999a typea comment-999.0
780-999a typea comment-999.0
781-999a typea comment-999.0
782-999a typea comment-999.0
783-999a typea comment-999.0
784-999a typea comment-999.0
785-999a typea comment-999.0
786-999a typea comment-999.0
787-999a typea comment-999.0
788-999a typea comment-999.0
789-999a typea comment-999.0
790-999a typea comment-999.0
791-999a typea comment-999.0
792-999a typea comment-999.0
793-999a typea comment-999.0
794-999a typea comment-999.0
795-999a typea comment-999.0
796-999a typea comment-999.0
797-999a typea comment-999.0
798-999a typea comment-999.0
799-999a typea comment-999.0
800-999a typea comment-999.0
801-999a typea comment-999.0
802-999a typea comment-999.0
803-999a typea comment-999.0
804-999a typea comment-999.0
805-999a typea comment-999.0
806-999a typea comment-999.0
807-999a typea comment-999.0
808-999a typea comment-999.0
809-999a typea comment-999.0
810-999a typea comment-999.0
811-999a typea comment-999.0
812-999a typea comment-999.0
813-999a typea comment-999.0
814-999a typea comment-999.0
815-999a typea comment-999.0
816-999a typea comment-999.0
817-999a typea comment-999.0
818-999a typea comment-999.0
819-999a typea comment-999.0
820-999a typea comment-999.0
821-999a typea comment-999.0
822-999a typea comment-999.0
823-999a typea comment-999.0
824-999a typea comment-999.0
825-999a typea comment-999.0
826-999a typea comment-999.0
827-999a typea comment-999.0
828-999a typea comment-999.0
829-999a typea comment-999.0
830-999a typea comment-999.0
831-999a typea comment-999.0
832-999a typea comment-999.0
833-999a typea comment-999.0
834-999a typea comment-999.0
835-999a typea comment-999.0
836-999a typea comment-999.0
837-999a typea comment-999.0
838-999a typea comment-999.0
839-999a typea comment-999.0
840-999a typea comment-999.0
841-999a typea comment-999.0
842-999a typea comment-999.0
843-999a typea comment-999.0
844-999a typea comment-999.0
845-999a typea comment-999.0
846-999a typea comment-999.0
847-999a typea comment-999.0
848-999a typea comment-999.0
849-999a typea comment-999.0
850-999a typea comment-999.0
851-999a typea comment-999.0
852-999a typea comment-999.0
853-999a typea comment-999.0
854-999a typea comment-999.0
855-999a typea comment-999.0
856-999a typea comment-999.0
857-999a typea comment-999.0
858-999a typea comment-999.0
859-999a typea comment-999.0
860-999a typea comment-999.0
861-999a typea comment-999.0
862-999a typea comment-999.0
863-999a typea comment-999.0
864-999a typea comment-999.0
865-999a typea comment-999.0
866-999a typea comment-999.0
867-999a typea comment-999.0
868-999a typea comment-999.0
869-999a typea comment-999.0
870-999a typea comment-999.0
871-999a typea comment-999.0
872-999a typea comment-999.0
873-999a typea comment-999.0
874-999a typea comment-999.0
875-999a typea comment-999.0
876-999a typea comment-999.0
877-999a typea comment-999.0
878-999a typea comment-999.0
879-999a typea comment-999.0
880-999a typea comment-999.0
881-999a typea comment-999.0
882-999a typea comment-999.0
883-999a typea comment-999.0
884-999a typea comment-999.0
885-999a typea comment-999.0
886-999a typea comment-999.0
887-999a typea comment-999.0
888-999a typea comment-999.0
889-999a typea comment-999.0
890-999a typea comment-999.0
891-999a typea comment-999.0
892-999a typea comment-999.0
893-999a typea comment-999.0
894-999a typea comment-999.0
895-999a typea comment-999.0
896-999a typea comment-999.0
897-999a typea comment-999.0
898-999a typea comment-999.0
899-999a typea comment-999.0
900-999a typea comment-999.0
901-999a typea comment-999.0
902-999a typea comment-999.0
903-999a typea comment-999.0
904-999a typea comment-999.0
905-999a typea comment-999.0
906-999a typea comment-999.0
907-999a typea comment-999.0
908-999a typea comment-999.0
909-999a typea comment-999.0
910-999a typea comment-999.0
911-999a typea comment-999.0
912-999a typea comment-999.0
913-999a typea comment-999.0
914-999a typea comment-999.0
915-999a typea comment-999.0
916-999a typea comment-999.0
917-999a typea comment-999.0
918-999a typea comment-999.0
919-999a typea comment-999.0
920-999a typea comment-999.0
921-999a typea comment-999.0
922-999a typea comment-999.0
923-999a typea comment-999.0
924-999a typea comment-999.0
925-999a typea comment-999.0
926-999a typea comment-999.0
927-999a typea comment-999.0
928-999a typea comment-999.0
929-999a typea comment-999.0
930-999a typea comment-999.0
931-999a typea comment-999.0
932-999a typea comment-999.0
933-999a typea comment-999.0
934-999a typea comment-999.0
935-999a typea comment-999.0
936-999a typea comment-999.0
937-999a typea comment-999.0
938-999a typea comment-999.0
939-999a typea comment-999.0
940-999a typea comment-999.0
941-999a typea comment-999.0
942-999a typea comment-999.0
943-999a typea comment-999.0
944-999a typea comment-999.0
945-999a typea comment-999.0
946-999a typea comment-999.0
947-999a typea comment-999.0
948-999a typea comment-999.0
949-999a typea comment-999.0
950-999a typea comment-999.0
951-999a typea comment-999.0
952-999a typea comment-999.0
953-999a typea comment-999.0
954-999a typea comment-999.0
955-999a typea comment-999.0
956-999a typea comment-999.0
957-999a typea comment-999.0
958-999a typea comment-999.0
959-999a typea comment-999.0
960-999a typea comment-999.0
961-999a typea comment-999.0
962-999a typea comment-999.0
963-999a typea comment-999.0
964-999a typea comment-999.0
965-999a typea comment-999.0
966-999a typea comment-999.0
967-999a typea comment-999.0
968-999a typea comment-999.0
969-999a typea comment-999.0
970-999a typea comment-999.0
971-999a typea comment-999.0
972-999a typea comment-999.0
973-999a typea comment-999.0
974-999a typea comment-999.0
975-999a typea comment-999.0
976-999a typea comment-999.0
977-999a typea comment-999.0
978-999a typea comment-999.0
979-999a typea comment-999.0
980-999a typea comment-999.0
981-999a typea comment-999.0
982-999a typea comment-999.0
983-999a typea comment-999.0
984-999a typea comment-999.0
985-999a typea comment-999.0
986-999a typea comment-999.0
987-999a typea comment-999.0
988-999a typea comment-999.0
989-999a typea comment-999.0
990-999a typea comment-999.0
991-999a typea comment-999.0
992-999a typea comment-999.0
993-999a typea comment-999.0
994-999a typea comment-999.0
995-999a typea comment-999.0
996-999a typea comment-999.0
997-999a typea comment-999.0
998-999a typea comment-999.0
999-999a typea comment-999.0
1000-999a typea comment-999.0
1001-999a typea comment-999.0
1002-999a typea comment-999.0
1003-999a typea comment-999.0
1004-999a typea comment-999.0
1005-999a typea comment-999.0
1006-999a typea comment-999.0
1007-999a typea comment-999.0
1008-999a typea comment-999.0
1009-999a typea comment-999.0
1010-999a typea comment-999.0
1011-999a typea comment-999.0
1012-999a typea comment-999.0
1013-999a typea comment-999.0
1014-999a typea comment-999.0
1015-999a typea comment-999.0
1016-999a typea comment-999.0
1017-999a typea comment-999.0
1018-999a typea comment-999.0
1019-999a typea comment-999.0
1020-999a typea comment-999.0
1021-999a typea comment-999.0
1022-999a typea comment-999.0
1023-999a typea comment-999.0
1024-999a typea comment-999.0
1025-999a typea comment-999.0
1026-999a typea comment-999.0
1027-999a typea comment-999.0
1028-999a typea comment-999.0
1029-999a typea comment-999.0
1030-999a typea comment-999.0
1031-999a typea comment-999.0
1032-999a typea comment-999.0
1033-999a typea comment-999.0
1034-999a typea comment-999.0
1035-999a typea comment-999.0
1036-999a typea comment-999.0
1037-999a typea comment-999.0
1038-999a typea comment-999.0
1039-999a typea comment-999.0
1040-999a typea comment-999.0
1041-999a typea comment-999.0
1042-999a typea comment-999.0
1043-999a typea comment-999.0
1044-999a typea comment-999.0
1045-999a typea comment-999.0
1046-999a typea comment-999.0
1047-999a typea comment-999.0
1048-999a typea comment-999.0
1049-999a typea comment-999.0
1050-999a typea comment-999.0
1051-999a typea comment-999.0
1052-999a typea comment-999.0
1053-999a typea comment-999.0
1054-999a typea comment-999.0
1055-999a typea comment-999.0
1056-999a typea comment-999.0
1057-999a typea comment-999.0
1058-999a typea comment-999.0
1059-999a typea comment-999.0
1060-999a typea comment-999.0
1061-999a typea comment-999.0
1062-999a typea comment-999.0
1063-999a typea comment-999.0
1064-999a typea comment-999.0
1065-999a typea comment-999.0
1066-999a typea comment-999.0
1067-999a typea comment-999.0
1068-999a typea comment-999.0
1069-999a typea comment-999.0
1070-999a typea comment-999.0
1071-999a typea comment-999.0
1072-999a typea comment-999.0
1073-999a typea comment-999.0
1074-999a typea comment-999.0
1075-999a typea comment-999.0
1076-999a typea comment-999.0
1077-999a typea comment-999.0
1078-999a typea comment-999.0
1079-999a typea comment-999.0
1080-999a typea comment-999.0
1081-999a typea comment-999.0
1082-999a typea comment-999.0
1083-999a typea comment-999.0
1084-999a typea comment-999.0
1085-999a typea comment-999.0
1086-999a typea comment-999.0
1087-999a typea comment-999.0
1088-999a typea comment-999.0
1089-999a typea comment-999.0
1090-999a typea comment-999.0
1091-999a typea comment-999.0
1092-999a typea comment-999.0
1093-999a typea comment-999.0
1094-999a typea comment-999.0
1095-999a typea comment-999.0
1096-999a typea comment-999.0
1097-999a typea comment-999.0
1098-999a typea comment-999.0
1099-999a typea comment-999.0
1100-999a typea comment-999.0
1101-999a typea comment-999.0
1102-999a typea comment-999.0
1103-999a typea comment-999.0
1104-999a typea comment-999.0
1105-999a typea comment-999.0
1106-999a typea comment-999.0
1107-999a typea comment-999.0
1108-999a typea comment-999.0
1109-999a typea comment-999.0
1110-999a typea comment-999.0
1111-999a typea comment-999.0
1112-999a typea comment-999.0
1113-999a typea comment-999.0
1114-999a typea comment-999.0
1115-999a typea comment-999.0
1116-999a typea comment-999.0
1117-999a typea comment-999.0
1118-999a typea comment-999.0
1119-999a typea comment-999.0
1120-999a typea comment-999.0
1121-999a typea comment-999.0
1122-999a typea comment-999.0
1123-999a typea comment-999.0
1124-999a typea comment-999.0
1125-999a typea comment-999.0
1126-999a typea comment-999.0
1127-999a typea comment-999.0
1128-999a typea comment-999.0
1129-999a typea comment-999.0
1130-999a typea comment-999.0
1131-999a typea comment-999.0
1132-999a typea comment-999.0
1133-999a typea comment-999.0
1134-999a typea comment-999.0
1135-999a typea comment-999.0
1136-999a typea comment-999.0
1137-999a typea comment-999.0
1138-999a typea comment-999.0
1139-999a typea comment-999.0
1140-999a typea comment-999.0
1141-999a typea comment-999.0
1142-999a typea comment-999.0
1143-999a typea comment-999.0
1144-999a typea comment-999.0
1145-999a typea comment-999.0
1146-999a typea comment-999.0
1147-999a typea comment-999.0
1148-999a typea comment-999.0
1149-999a typea comment-999.0
1150-999a typea comment-999.0
1151-999a typea comment-999.0
1152-999a typea comment-999.0
1153-999a typea comment-999.0
1154-999a typea comment-999.0
1155-999a typea comment-999.0
1156-999a typea comment-999.0
1157-999a typea comment-999.0
1158-999a typea comment-999.0
1159-999a typea comment-999.0
1160-999a typea comment-999.0
1161-999a typea comment-999.0
1162-999a typea comment-999.0
1163-999a typea comment-999.0
1164-999a typea comment-999.0
1165-999a typea comment-999.0
1166-999a typea comment-999.0
1167-999a typea comment-999.0
1168-999a typea comment-999.0
1169-999a typea comment-999.0
1170-999a typea comment-999.0
1171-999a typea comment-999.0
1172-999a typea comment-999.0
1173-999a typea comment-999.0
1174-999a typea comment-999.0
1175-999a typea comment-999.0
1176-999a typea comment-999.0
1177-999a typea comment-999.0
1178-999a typea comment-999.0
1179-999a typea comment-999.0
1180-999a typea comment-999.0
1181-999a typea comment-999.0
1182-999a typea comment-999.0
1183-999a typea comment-999.0
1184-999a typea comment-999.0
1185-999a typea comment-999.0
1186-999a typea comment-999.0
1187-999a typea comment-999.0
1188-999a typea comment-999.0
1189-999a typea comment-999.0
1190-999a typea comment-999.0
1191-999a typea comment-999.0
1192-999a typea comment-999.0
1193-999a typea comment-999.0
1194-999a typea comment-999.0
1195-999a typea comment-999.0
1196-999a typea comment-999.0
1197-999a typea comment-999.0
1198-999a typea comment-999.0
1199-999a typea comment-999.0
1200-999a typea comment-999.0
1201-999a typea comment-999.0
1202-999a typea comment-999.0
1203-999a typea comment-999.0
1204-999a typea comment-999.0
1205-999a typea comment-999.0
1206-999a typea comment-999.0
1207-999a typea comment-999.0
1208-999a typea comment-999.0
1209-999a typea comment-999.0
1210-999a typea comment-999.0
1211-999a typea comment-999.0
1212-999a typea comment-999.0
1213-999a typea comment-999.0
1214-999a typea comment-999.0
1215-999a typea comment-999.0
1216-999a typea comment-999.0
1217-999a typea comment-999.0
1218-999a typea comment-999.0
1219-999a typea comment-999.0
1220-999a typea comment-999.0
1221-999a typea comment-999.0
1222-999a typea comment-999.0
1223-999a typea comment-999.0
1224-999a typea comment-999.0
1225-999a typea comment-999.0
1226-999a typea comment-999.0
1227-999a typea comment-999.0
1228-999a typea comment-999.0
1229-999a typea comment-999.0
1230-999a typea comment-999.0
1231-999a typea comment-999.0
1232-999a typea comment-999.0
1233-999a typea comment-999.0
1234-999a typea comment-999.0
1235-999a typea comment-999.0
1236-999a typea comment-999.0
1237-999a typea comment-999.0
1238-999a typea comment-999.0
1239-999a typea comment-999.0
1240-999a typea comment-999.0
1241-999a typea comment-999.0
1242-999a typea comment-999.0
1243-999a typea comment-999.0
1244-999a typea comment-999.0
1245-999a typea comment-999.0
1246-999a typea comment-999.0
1247-999a typea comment-999.0
1248-999a typea comment-999.0
1249-999a typea comment-999.0
1250-999a typea comment-999.0
1251-999a typea comment-999.0
1252-999a typea comment-999.0
1253-999a typea comment-999.0
1254-999a typea comment-999.0
1255-999a typea comment-999.0
1256-999a typea comment-999.0
1257-999a typea comment-999.0
1258-999a typea comment-999.0
1259-999a typea comment-999.0
1260-999a typea comment-999.0
1261-999a typea comment-999.0
1262-999a typea comment-999.0
1263-999a typea comment-999.0
1264-999a typea comment-999.0
1265-999a typea comment-999.0
1266-999a typea comment-999.0
1267-999a typea comment-999.0
1268-999a typea comment-999.0
1269-999a typea comment-999.0
1270-999a typea comment-999.0
1271-999a typea comment-999.0
1272-999a typea comment-999.0
1273-999a typea comment-999.0
1274-999a typea comment-999.0
1275-999a typea comment-999.0
1276-999a typea comment-999.0
1277-999a typea comment-999.0
1278-999a typea comment-999.0
1279-999a typea comment-999.0
1280-999a typea comment-999.0
1281-999a typea comment-999.0
1282-999a typea comment-999.0
1283-999a typea comment-999.0
1284-999a typea comment-999.0
1285-999a typea comment-999.0
1286-999a typea comment-999.0
1287-999a typea comment-999.0
1288-999a typea comment-999.0
1289-999a typea comment-999.0
1290-999a typea comment-999.0
1291-999a typea comment-999.0
1292-999a typea comment-999.0
1293-999a typea comment-999.0
1294-999a typea comment-999.0
1295-999a typea comment-999.0
1296-999a typea comment-999.0
1297-999a typea comment-999.0
1298-999a typea comment-999.0
1299-999a typea comment-999.0
1300-999a typea comment-999.0
1301-999a typea comment-999.0
1302-999a typea comment-999.0
1303-999a typea comment-999.0
1304-999a typea comment-999.0
1305-999a typea comment-999.0
1306-999a typea comment-999.0
1307-999a typea comment-999.0
1308-999a typea comment-999.0
1309-999a typea comment-999.0
1310-999a typea comment-999.0
1311-999a typea comment-999.0
1312-999a typea comment-999.0
1313-999a typea comment-999.0
1314-999a typea comment-999.0
1315-999a typea comment-999.0
1316-999a typea comment-999.0
1317-999a typea comment-999.0
1318-999a typea comment-999.0
1319-999a typea comment-999.0
1320-999a typea comment-999.0
1321-999a typea comment-999.0
1322-999a typea comment-999.0
1323-999a typea comment-999.0
1324-999a typea comment-999.0
1325-999a typea comment-999.0
1326-999a typea comment-999.0
1327-999a typea comment-999.0
1328-999a typea comment-999.0
1329-999a typea comment-999.0
1330-999a typea comment-999.0
1331-999a typea comment-999.0
1332-999a typea comment-999.0
1333-999a typea comment-999.0
1334-999a typea comment-999.0
1335-999a typea comment-999.0
1336-999a typea comment-999.0
1337-999a typea comment-999.0
1338-999a typea comment-999.0
1339-999a typea comment-999.0
1340-999a typea comment-999.0
1341-999a typea comment-999.0
1342-999a typea comment-999.0
1343-999a typea comment-999.0
1344-999a typea comment-999.0
1345-999a typea comment-999.0
1346-999a typea comment-999.0
1347-999a typea comment-999.0
1348-999a typea comment-999.0
1349-999a typea comment-999.0
1350-999a typea comment-999.0
1351-999a typea comment-999.0
1352-999a typea comment-999.0
1353-999a typea comment-999.0
1354-999a typea comment-999.0
1355-999a typea comment-999.0
1356-999a typea comment-999.0
1357-999a typea comment-999.0
1358-999a typea comment-999.0
1359-999a typea comment-999.0
1360-999a typea comment-999.0
1361-999a typea comment-999.0
1362-999a typea comment-999.0
1363-999a typea comment-999.0
1364-999a typea comment-999.0
1365-999a typea comment-999.0
1366-999a typea comment-999.0
1367-999a typea comment-999.0
1368-999a typea comment-999.0
1369-999a typea comment-999.0
1370-999a typea comment-999.0
1371-999a typea comment-999.0
1372-999a typea comment-999.0
1373-999a typea comment-999.0
1374-999a typea comment-999.0
1375-999a typea comment-999.0
1376-999a typea comment-999.0
1377-999a typea comment-999.0
1378-999a typea comment-999.0
1379-999a typea comment-999.0
1380-999a typea comment-999.0
1381-999a typea comment-999.0
1382-999a typea comment-999.0
1383-999a typea comment-999.0
1384-999a typea comment-999.0
1385-999a typea comment-999.0
1386-999a typea comment-999.0
1387-999a typea comment-999.0
1388-999a typea comment-999.0
1389-999a typea comment-999.0
1390-999a typea comment-999.0
1391-999a typea comment-999.0
1392-999a typea comment-999.0
1393-999a typea comment-999.0
1394-999a typea comment-999.0
1395-999a typea comment-999.0
1396-999a typea comment-999.0
1397-999a typea comment-999.0
1398-999a typea comment-999.0
1399-999a typea comment-999.0
1400-999a typea comment-999.0
1401-999a typea comment-999.0
1402-999a typea comment-999.0
1403-999a typea comment-999.0
1404-999a typea comment-999.0
1405-999a typea comment-999.0
1406-999a typea comment-999.0
1407-999a typea comment-999.0
1408-999a typea comment-999.0
1409-999a typea comment-999.0
1410-999a typea comment-999.0
1411-999a typea comment-999.0
1412-999a typea comment-999.0
1413-999a typea comment-999.0
1414-999a typea comment-999.0
1415-999a typea comment-999.0
1416-999a typea comment-999.0
1417-999a typea comment-999.0
1418-999a typea comment-999.0
1419-999a typea comment-999.0
1420-999a typea comment-999.0
1421-999a typea comment-999.0
1422-999a typea comment-999.0
1423-999a typea comment-999.0
1424-999a typea comment-999.0
1425-999a typea comment-999.0
1426-999a typea comment-999.0
1427-999a typea comment-999.0
1428-999a typea comment-999.0
1429-999a typea comment-999.0
1430-999a typea comment-999.0
1431-999a typea comment-999.0
1432-999a typea comment-999.0
1433-999a typea comment-999.0
1434-999a typea comment-999.0
1435-999a typea comment-999.0
1436-999a typea comment-999.0
1437-999a typea comment-999.0
1438-999a typea comment-999.0
1439-999a typea comment-999.0
1440-999a typea comment-999.0
1441-999a typea comment-999.0
1442-999a typea comment-999.0
1443-999a typea comment-999.0
1444-999a typea comment-999.0
1445-999a typea comment-999.0
1446-999a typea comment-999.0
1447-999a typea comment-999.0
1448-999a typea comment-999.0
1449-999a typea comment-999.0
1450-999a typea comment-999.0
1451-999a typea comment-999.0
1452-999a typea comment-999.0
1453-999a typea comment-999.0
1454-999a typea comment-999.0
1455-999a typea comment-999.0
1456-999a typea comment-999.0
1457-999a typea comment-999.0
1458-999a typea comment-999.0
1459-999a typea comment-999.0
1460-999a typea comment-999.0
1461-999a typea comment-999.0
1462-999a typea comment-999.0
1463-999a typea comment-999.0
1464-999a typea comment-999.0
1465-999a typea comment-999.0
1466-999a typea comment-999.0
1467-999a typea comment-999.0
1468-999a typea comment-999.0
1469-999a typea comment-999.0
1470-999a typea comment-999.0
1471-999a typea comment-999.0
1472-999a typea comment-999.0
1473-999a typea comment-999.0
1474-999a typea comment-999.0
1475-999a typea comment-999.0
1476-999a typea comment-999.0
1477-999a typea comment-999.0
1478-999a typea comment-999.0
1479-999a typea comment-999.0
1480-999a typea comment-999.0
1481-999a typea comment-999.0
1482-999a typea comment-999.0
1483-999a typea comment-999.0
1484-999a typea comment-999.0
1485-999a typea comment-999.0
1486-999a typea comment-999.0
1487-999a typea comment-999.0
1488-999a typea comment-999.0
1489-999a typea comment-999.0
1490-999a typea comment-999.0
1491-999a typea comment-999.0
1492-999a typea comment-999.0
1493-999a typea comment-999.0
1494-999a typea comment-999.0
1495-999a typea comment-999.0
1496-999a typea comment-999.0
1497-999a typea comment-999.0
1498-999a typea comment-999.0
1499-999a typea comment-999.0
1500-999a typea comment-999.0
1501-999a typea comment-999.0
1502-999a typea comment-999.0
1503-999a typea comment-999.0
1504-999a typea comment-999.0
1505-999a typea comment-999.0
1506-999a typea comment-999.0
1507-999a typea comment-999.0
1508-999a typea comment-999.0
1509-999a typea comment-999.0
1510-999a typea comment-999.0
1511-999a typea comment-999.0
1512-999a typea comment-999.0
1513-999a typea comment-999.0
1514-999a typea comment-999.0
1515-999a typea comment-999.0
1516-999a typea comment-999.0
1517-999a typea comment-999.0
1518-999a typea comment-999.0
1519-999a typea comment-999.0
1520-999a typea comment-999.0
1521-999a typea comment-999.0
1522-999a typea comment-999.0
1523-999a typea comment-999.0
1524-999a typea comment-999.0
1525-999a typea comment-999.0
1526-999a typea comment-999.0
1527-999a typea comment-999.0
1528-999a typea comment-999.0
1529-999a typea comment-999.0
1530-999a typea comment-999.0
1531-999a typea comment-999.0
1532-999a typea comment-999.0
1533-999a typea comment-999.0
1534-999a typea comment-999.0
1535-999a typea comment-999.0
1536-999a typea comment-999.0
1537-999a typea comment-999.0
1538-999a typea comment-999.0
1539-999a typea comment-999.0
1540-999a typea comment-999.0
1541-999a typea comment-999.0
1542-999a typea comment-999.0
1543-999a typea comment-999.0
1544-999a typea comment-999.0
1545-999a typea comment-999.0
1546-999a typea comment-999.0
1547-999a typea comment-999.0
1548-999a typea comment-999.0
1549-999a typea comment-999.0
1550-999a typea comment-999.0
1551-999a typea comment-999.0
1552-999a typea comment-999.0
1553-999a typea comment-999.0
1554-999a typea comment-999.0
1555-999a typea comment-999.0
1556-999a typea comment-999.0
1557-999a typea comment-999.0
1558-999a typea comment-999.0
1559-999a typea comment-999.0
1560-999a typea comment-999.0
1561-999a typea comment-999.0
1562-999a typea comment-999.0
1563-999a typea comment-999.0
1564-999a typea comment-999.0
1565-999a typea comment-999.0
1566-999a typea comment-999.0
1567-999a typea comment-999.0
1568-999a typea comment-999.0
1569-999a typea comment-999.0
1570-999a typea comment-999.0
1571-999a typea comment-999.0
1572-999a typea comment-999.0
1573-999a typea comment-999.0
1574-999a typea comment-999.0
1575-999a typea comment-999.0
1576-999a typea comment-999.0
1577-999a typea comment-999.0
1578-999a typea comment-999.0
1579-999a typea comment-999.0
1580-999a typea comment-999.0
1581-999a typea comment-999.0
1582-999a typea comment-999.0
1583-999a typea comment-999.0
1584-999a typea comment-999.0
1585-999a typea comment-999.0
1586-999a typea comment-999.0
1587-999a typea comment-999.0
1588-999a typea comment-999.0
1589-999a typea comment-999.0
1590-999a typea comment-999.0
1591-999a typea comment-999.0
1592-999a typea comment-999.0
1593-999a typea comment-999.0
1594-999a typea comment-999.0
1595-999a typea comment-999.0
1596-999a typea comment-999.0
1597-999a typea comment-999.0
1598-999a typea comment-999.0
1599-999a typea comment-999.0
1600-999a typea comment-999.0
1601-999a typea comment-999.0
1602-999a typea comment-999.0
1603-999a typea comment-999.0
1604-999a typea comment-999.0
1605-999a typea comment-999.0
1606-999a typea comment-999.0
1607-999a typea comment-999.0
1608-999a typea comment-999.0
1609-999a typea comment-999.0
1610-999a typea comment-999.0
1611-999a typea comment-999.0
1612-999a typea comment-999.0
1613-999a typea comment-999.0
1614-999a typea comment-999.0
1615-999a typea comment-999.0
1616-999a typea comment-999.0
1617-999a typea comment-999.0
1618-999a typea comment-999.0
1619-999a typea comment-999.0
1620-999a typea comment-999.0
1621-999a typea comment-999.0
1622-999a typea comment-999.0
1623-999a typea comment-999.0
1624-999a typea comment-999.0
1625-999a typea comment-999.0
1626-999a typea comment-999.0
1627-999a typea comment-999.0
1628-999a typea comment-999.0
1629-999a typea comment-999.0
1630-999a typea comment-999.0
1631-999a typea comment-999.0
1632-999a typea comment-999.0
1633-999a typea comment-999.0
1634-999a typea comment-999.0
1635-999a typea comment-999.0
1636-999a typea comment-999.0
1637-999a typea comment-999.0
1638-999a typea comment-999.0
1639-999a typea comment-999.0
1640-999a typea comment-999.0
1641-999a typea comment-999.0
1642-999a typea comment-999.0
1643-999a typea comment-999.0
1644-999a typea comment-999.0
1645-999a typea comment-999.0
1646-999a typea comment-999.0
1647-999a typea comment-999.0
1648-999a typea comment-999.0
1649-999a typea comment-999.0
1650-999a typea comment-999.0
1651-999a typea comment-999.0
1652-999a typea comment-999.0
1653-999a typea comment-999.0
1654-999a typea comment-999.0
1655-999a typea comment-999.0
1656-999a typea comment-999.0
1657-999a typea comment-999.0
1658-999a typea comment-999.0
1659-999a typea comment-999.0
1660-999a typea comment-999.0
1661-999a typea comment-999.0
1662-999a typea comment-999.0
1663-999a typea comment-999.0
1664-999a typea comment-999.0
1665-999a typea comment-999.0
1666-999a typea comment-999.0
1667-999a typea comment-999.0
1668-999a typea comment-999.0
1669-999a typea comment-999.0
1670-999a typea comment-999.0
1671-999a typea comment-999.0
1672-999a typea comment-999.0
1673-999a typea comment-999.0
1674-999a typea comment-999.0
1675-999a typea comment-999.0
1676-999a typea comment-999.0
1677-999a typea comment-999.0
1678-999a typea comment-999.0
1679-999a typea comment-999.0
1680-999a typea comment-999.0
1681-999a typea comment-999.0
1682-999a typea comment-999.0
1683-999a typea comment-999.0
1684-999a typea comment-999.0
1685-999a typea comment-999.0
1686-999a typea comment-999.0
1687-999a typea comment-999.0
1688-999a typea comment-999.0
1689-999a typea comment-999.0
1690-999a typea comment-999.0
1691-999a typea comment-999.0
1692-999a typea comment-999.0
1693-999a typea comment-999.0
1694-999a typea comment-999.0
1695-999a typea comment-999.0
1696-999a typea comment-999.0
1697-999a typea comment-999.0
1698-999a typea comment-999.0
1699-999a typea comment-999.0
1700-999a typea comment-999.0
1701-999a typea comment-999.0
1702-999a typea comment-999.0
1703-999a typea comment-999.0
1704-999a typea comment-999.0
1705-999a typea comment-999.0
1706-999a typea comment-999.0
1707-999a typea comment-999.0
1708-999a typea comment-999.0
1709-999a typea comment-999.0
1710-999a typea comment-999.0
1711-999a typea comment-999.0
1712-999a typea comment-999.0
1713-999a typea comment-999.0
1714-999a typea comment-999.0
1715-999a typea comment-999.0
1716-999a typea comment-999.0
1717-999a typea comment-999.0
1718-999a typea comment-999.0
1719-999a typea comment-999.0
1720-999a typea comment-999.0
1721-999a typea comment-999.0
1722-999a typea comment-999.0
1723-999a typea comment-999.0
1724-999a typea comment-999.0
1725-999a typea comment-999.0
1726-999a typea comment-999.0
1727-999a typea comment-999.0
1728-999a typea comment-999.0
1729-999a typea comment-999.0
1730-999a typea comment-999.0
1731-999a typea comment-999.0
1732-999a typea comment-999.0
1733-999a typea comment-999.0
1734-999a typea comment-999.0
1735-999a typea comment-999.0
1736-999a typea comment-999.0
1737-999a typea comment-999.0
1738-999a typea comment-999.0
1739-999a typea comment-999.0
1740-999a typea comment-999.0
1741-999a typea comment-999.0
1742-999a typea comment-999.0
1743-999a typea comment-999.0
1744-999a typea comment-999.0
1745-999a typea comment-999.0
1746-999a typea comment-999.0
1747-999a typea comment-999.0
1748-999a typea comment-999.0
1749-999a typea comment-999.0
1750-999a typea comment-999.0
1751-999a typea comment-999.0
1752-999a typea comment-999.0
1753-999a typea comment-999.0
1754-999a typea comment-999.0
1755-999a typea comment-999.0
1756-999a typea comment-999.0
1757-999a typea comment-999.0
1758-999a typea comment-999.0
1759-999a typea comment-999.0
1760-999a typea comment-999.0
1761-999a typea comment-999.0
1762-999a typea comment-999.0
1763-999a typea comment-999.0
1764-999a typea comment-999.0
1765-999a typea comment-999.0
1766-999a typea comment-999.0
1767-999a typea comment-999.0
1768-999a typea comment-999.0
1769-999a typea comment-999.0
1770-999a typea comment-999.0
1771-999a typea comment-999.0
1772-999a typea comment-999.0
1773-999a typea comment-999.0
1774-999a typea comment-999.0
1775-999a typea comment-999.0
1776-999a typea comment-999.0
1777-999a typea comment-999.0
1778-999a typea comment-999.0
1779-999a typea comment-999.0
1780-999a typea comment-999.0
1781-999a typea comment-999.0
1782-999a typea comment-999.0
1783-999a typea comment-999.0
1784-999a typea comment-999.0
1785-999a typea comment-999.0
1786-999a typea comment-999.0
1787-999a typea comment-999.0
1788-999a typea comment-999.0
1789-999a typea comment-999.0
1790-999a typea comment-999.0
1791-999a typea comment-999.0
1792-999a typea comment-999.0
1793-999a typea comment-999.0
1794-999a typea comment-999.0
1795-999a typea comment-999.0
1796-999a typea comment-999.0
1797-999a typea comment-999.0
1798-999a typea comment-999.0
1799-999a typea comment-999.0
1800-999a typea comment-999.0
1801-999a typea comment-999.0
1802-999a typea comment-999.0
1803-999a typea comment-999.0
1804-999a typea comment-999.0
1805-999a typea comment-999.0
1806-999a typea comment-999.0
1807-999a typea comment-999.0
1808-999a typea comment-999.0
1809-999a typea comment-999.0
1810-999a typea comment-999.0
1811-999a typea comment-999.0
1812-999a typea comment-999.0
1813-999a typea comment-999.0
1814-999a typea comment-999.0
1815-999a typea comment-999.0
1816-999a typea comment-999.0
1817-999a typea comment-999.0
1818-999a typea comment-999.0
1819-999a typea comment-999.0
1820-999a typea comment-999.0
1821-999a typea comment-999.0
1822-999a typea comment-999.0
1823-999a typea comment-999.0
1824-999a typea comment-999.0
1825-999a typea comment-999.0
1826-999a typea comment-999.0
1827-999a typea comment-999.0
1828-999a typea comment-999.0
1829-999a typea comment-999.0
1830-999a typea comment-999.0
1831-999a typea comment-999.0
1832-999a typea comment-999.0
1833-999a typea comment-999.0
1834-999a typea comment-999.0
1835-999a typea comment-999.0
1836-999a typea comment-999.0
1837-999a typea comment-999.0
1838-999a typea comment-999.0
1839-999a typea comment-999.0
1840-999a typea comment-999.0
1841-999a typea comment-999.0
1842-999a typea comment-999.0
1843-999a typea comment-999.0
1844-999a typea comment-999.0
1845-999a typea comment-999.0
1846-999a typea comment-999.0
1847-999a typea comment-999.0
1848-999a typea comment-999.0
1849-999a typea comment-999.0
1850-999a typea comment-999.0
1851-999a typea comment-999.0
1852-999a typea comment-999.0
1853-999a typea comment-999.0
1854-999a typea comment-999.0
1855-999a typea comment-999.0
1856-999a typea comment-999.0
1857-999a typea comment-999.0
1858-999a typea comment-999.0
1859-999a typea comment-999.0
1860-999a typea comment-999.0
1861-999a typea comment-999.0
1862-999a typea comment-999.0
1863-999a typea comment-999.0
1864-999a typea comment-999.0
1865-999a typea comment-999.0
1866-999a typea comment-999.0
1867-999a typea comment-999.0
1868-999a typea comment-999.0
1869-999a typea comment-999.0
1870-999a typea comment-999.0
1871-999a typea comment-999.0
1872-999a typea comment-999.0
1873-999a typea comment-999.0
1874-999a typea comment-999.0
1875-999a typea comment-999.0
1876-999a typea comment-999.0
1877-999a typea comment-999.0
1878-999a typea comment-999.0
1879-999a typea comment-999.0
1880-999a typea comment-999.0
1881-999a typea comment-999.0
1882-999a typea comment-999.0
1883-999a typea comment-999.0
1884-999a typea comment-999.0
1885-999a typea comment-999.0
1886-999a typea comment-999.0
1887-999a typea comment-999.0
1888-999a typea comment-999.0
1889-999a typea comment-999.0
1890-999a typea comment-999.0
1891-999a typea comment-999.0
1892-999a typea comment-999.0
1893-999a typea comment-999.0
1894-999a typea comment-999.0
1895-999a typea comment-999.0
1896-999a typea comment-999.0
1897-999a typea comment-999.0
1898-999a typea comment-999.0
1899-999a typea comment-999.0
1900-999a typea comment-999.0
1901-999a typea comment-999.0
1902-999a typea comment-999.0
1903-999a typea comment-999.0
1904-999a typea comment-999.0
1905-999a typea comment-999.0
1906-999a typea comment-999.0
1907-999a typea comment-999.0
1908-999a typea comment-999.0
1909-999a typea comment-999.0
1910-999a typea comment-999.0
1911-999a typea comment-999.0
1912-999a typea comment-999.0
1913-999a typea comment-999.0
1914-999a typea comment-999.0
1915-999a typea comment-999.0
1916-999a typea comment-999.0
1917-999a typea comment-999.0
1918-999a typea comment-999.0
1919-999a typea comment-999.0
1920-999a typea comment-999.0
1921-999a typea comment-999.0
1922-999a typea comment-999.0
1923-999a typea comment-999.0
1924-999a typea comment-999.0
1925-999a typea comment-999.0
1926-999a typea comment-999.0
1927-999a typea comment-999.0
1928-999a typea comment-999.0
1929-999a typea comment-999.0
1930-999a typea comment-999.0
1931-999a typea comment-999.0
1932-999a typea comment-999.0
1933-999a typea comment-999.0
1934-999a typea comment-999.0
1935-999a typea comment-999.0
1936-999a typea comment-999.0
1937-999a typea comment-999.0
1938-999a typea comment-999.0
1939-999a typea comment-999.0
1940-999a typea comment-999.0
1941-999a typea comment-999.0
1942-999a typea comment-999.0
1943-999a typea comment-999.0
1944-999a typea comment-999.0
1945-999a typea comment-999.0
1946-999a typea comment-999.0
1947-999a typea comment-999.0
1948-999a typea comment-999.0
1949-999a typea comment-999.0
1950-999a typea comment-999.0
1951-999a typea comment-999.0
1952-999a typea comment-999.0
1953-999a typea comment-999.0
1954-999a typea comment-999.0
1955-999a typea comment-999.0
1956-999a typea comment-999.0
1957-999a typea comment-999.0
1958-999a typea comment-999.0
1959-999a typea comment-999.0
1960-999a typea comment-999.0
1961-999a typea comment-999.0
1962-999a typea comment-999.0
1963-999a typea comment-999.0
1964-999a typea comment-999.0
1965-999a typea comment-999.0
1966-999a typea comment-999.0
1967-999a typea comment-999.0
1968-999a typea comment-999.0
1969-999a typea comment-999.0
1970-999a typea comment-999.0
1971-999a typea comment-999.0
1972-999a typea comment-999.0
1973-999a typea comment-999.0
1974-999a typea comment-999.0
1975-999a typea comment-999.0
1976-999a typea comment-999.0
1977-999a typea comment-999.0
1978-999a typea comment-999.0
1979-999a typea comment-999.0
1980-999a typea comment-999.0
1981-999a typea comment-999.0
1982-999a typea comment-999.0
1983-999a typea comment-999.0
1984-999a typea comment-999.0
1985-999a typea comment-999.0
1986-999a typea comment-999.0
1987-999a typea comment-999.0
1988-999a typea comment-999.0
1989-999a typea comment-999.0
1990-999a typea comment-999.0
1991-999a typea comment-999.0
1992-999a typea comment-999.0
1993-999a typea comment-999.0
1994-999a typea comment-999.0
1995-999a typea comment-999.0
1996-999a typea comment-999.0
1997-999a typea comment-999.0
1998-999a typea comment-999.0
1999-999a typea comment-999.0
2000-999a typea comment-999.0
2001-999a typea comment-999.0
2002-999a typea comment-999.0
2003-999a typea comment-999.0
2004-999a typea comment-999.0
2005-999a typea comment-999.0
2006-999a typea comment-999.0
2007-999a typea comment-999.0
2008-999a typea comment-999.0
2009-999a typea comment-999.0
2010-999a typea comment-999.0
2011-999a typea comment-999.0
2012-999a typea comment-999.0
2013-999a typea comment-999.0
2014-999a typea comment-999.0
2015-999a typea comment-999.0
2016-999a typea comment-999.0
2017-999a typea comment-999.0
2018-999a typea comment-999.0
2019-999a typea comment-999.0
2020-999a typea comment-999.0
2021-999a typea comment-999.0
2022-999a typea comment-999.0
2023-999a typea comment-999.0
2024-999a typea comment-999.0
2025-999a typea comment-999.0
2026-999a typea comment-999.0
2027-999a typea comment-999.0
2028-999a typea comment-999.0
2029-999a typea comment-999.0
2030-999a typea comment-999.0
2031-999a typea comment-999.0
2032-999a typea comment-999.0
2033-999a typea comment-999.0
2034-999a typea comment-999.0
2035-999a typea comment-999.0
2036-999a typea comment-999.0
2037-999a typea comment-999.0
2038-999a typea comment-999.0
2039-999a typea comment-999.0
2040-999a typea comment-999.0
2041-999a typea comment-999.0
2042-999a typea comment-999.0
2043-999a typea comment-999.0
2044-999a typea comment-999.0
2045-999a typea comment-999.0
2046-999a typea comment-999.0
2047-999a typea comment-999.0
2048-999a typea comment-999.0
2049-999a typea comment-999.0
2050-999a typea comment-999.0
2051-999a typea comment-999.0
2052-999a typea comment-999.0
2053-999a typea comment-999.0
2054-999a typea comment-999.0
2055-999a typea comment-999.0
2056-999a typea comment-999.0
2057-999a typea comment-999.0
2058-999a typea comment-999.0
2059-999a typea comment-999.0
2060-999a typea comment-999.0
2061-999a typea comment-999.0
2062-999a typea comment-999.0
2063-999a typea comment-999.0
2064-999a typea comment-999.0
2065-999a typea comment-999.0
2066-999a typea comment-999.0
2067-999a typea comment-999.0
2068-999a typea comment-999.0
2069-999a typea comment-999.0
2070-999a typea comment-999.0
2071-999a typea comment-999.0
2072-999a typea comment-999.0
2073-999a typea comment-999.0
2074-999a typea comment-999.0
2075-999a typea comment-999.0
2076-999a typea comment-999.0
2077-999a typea comment-999.0
2078-999a typea comment-999.0
2079-999a typea comment-999.0
2080-999a typea comment-999.0
2081-999a typea comment-999.0
2082-999a typea comment-999.0
2083-999a typea comment-999.0
2084-999a typea comment-999.0
2085-999a typea comment-999.0
2086-999a typea comment-999.0
2087-999a typea comment-999.0
2088-999a typea comment-999.0
2089-999a typea comment-999.0
2090-999a typea comment-999.0
2091-999a typea comment-999.0
2092-999a typea comment-999.0
2093-999a typea comment-999.0
2094-999a typea comment-999.0
2095-999a typea comment-999.0
2096-999a typea comment-999.0
2097-999a typea comment-999.0
2098-999a typea comment-999.0
2099-999a typea comment-999.0
2100-999a typea comment-999.0
2101-999a typea comment-999.0
2102-999a typea comment-999.0
2103-999a typea comment-999.0
2104-999a typea comment-999.0
2105-999a typea comment-999.0
2106-999a typea comment-999.0
2107-999a typea comment-999.0
2108-999a typea comment-999.0
2109-999a typea comment-999.0
2110-999a typea comment-999.0
2111-999a typea comment-999.0
2112-999a typea comment-999.0
2113-999a typea comment-999.0
2114-999a typea comment-999.0
2115-999a typea comment-999.0
2116-999a typea comment-999.0
2117-999a typea comment-999.0
2118-999a typea comment-999.0
2119-999a typea comment-999.0
2120-999a typea comment-999.0
2121-999a typea comment-999.0
2122-999a typea comment-999.0
2123-999a typea comment-999.0
2124-999a typea comment-999.0
2125-999a typea comment-999.0
2126-999a typea comment-999.0
2127-999a typea comment-999.0
2128-999a typea comment-999.0
2129-999a typea comment-999.0
2130-999a typea comment-999.0
2131-999a typea comment-999.0
2132-999a typea comment-999.0
2133-999a typea comment-999.0
2134-999a typea comment-999.0
2135-999a typea comment-999.0
2136-999a typea comment-999.0
2137-999a typea comment-999.0
2138-999a typea comment-999.0
2139-999a typea comment-999.0
2140-999a typea comment-999.0
2141-999a typea comment-999.0
2142-999a typea comment-999.0
2143-999a typea comment-999.0
2144-999a typea comment-999.0
2145-999a typea comment-999.0
2146-999a typea comment-999.0
2147-999a typea comment-999.0
2148-999a typea comment-999.0
2149-999a typea comment-999.0
2150-999a typea comment-999.0
2151-999a typea comment-999.0
2152-999a typea comment-999.0
2153-999a typea comment-999.0
2154-999a typea comment-999.0
2155-999a typea comment-999.0
2156-999a typea comment-999.0
2157-999a typea comment-999.0
2158-999a typea comment-999.0
2159-999a typea comment-999.0
2160-999a typea comment-999.0
2161-999a typea comment-999.0
2162-999a typea comment-999.0
2163-999a typea comment-999.0
2164-999a typea comment-999.0
2165-999a typea comment-999.0
2166-999a typea comment-999.0
2167-999a typea comment-999.0
2168-999a typea comment-999.0
2169-999a typea comment-999.0
2170-999a typea comment-999.0
2171-999a typea comment-999.0
2172-999a typea comment-999.0
2173-999a typea comment-999.0
2174-999a typea comment-999.0
2175-999a typea comment-999.0
2176-999a typea comment-999.0
2177-999a typea comment-999.0
2178-999a typea comment-999.0
2179-999a typea comment-999.0
2180-999a typea comment-999.0
2181-999a typea comment-999.0
2182-999a typea comment-999.0
2183-999a typea comment-999.0
2184-999a typea comment-999.0
2185-999a typea comment-999.0
2186-999a typea comment-999.0
2187-999a typea comment-999.0
2188-999a typea comment-999.0
2189-999a typea comment-999.0
2190-999a typea comment-999.0
2191-999a typea comment-999.0
2192-999a typea comment-999.0
2193-999a typea comment-999.0
2194-999a typea comment-999.0
2195-999a typea comment-999.0
2196-999a typea comment-999.0
2197-999a typea comment-999.0
2198-999a typea comment-999.0
2199-999a typea comment-999.0
2200-999a typea comment-999.0
2201-999a typea comment-999.0
2202-999a typea comment-999.0
2203-999a typea comment-999.0
2204-999a typea comment-999.0
2205-999a typea comment-999.0
2206-999a typea comment-999.0
2207-999a typea comment-999.0
2208-999a typea comment-999.0
2209-999a typea comment-999.0
2210-999a typea comment-999.0
2211-999a typea comment-999.0
2212-999a typea comment-999.0
2213-999a typea comment-999.0
2214-999a typea comment-999.0
2215-999a typea comment-999.0
2216-999a typea comment-999.0
2217-999a typea comment-999.0
2218-999a typea comment-999.0
2219-999a typea comment-999.0
2220-999a typea comment-999.0
2221-999a typea comment-999.0
2222-999a typea comment-999.0
2223-999a typea comment-999.0
2224-999a typea comment-999.0
2225-999a typea comment-999.0
2226-999a typea comment-999.0
2227-999a typea comment-999.0
2228-999a typea comment-999.0
2229-999a typea comment-999.0
2230-999a typea comment-999.0
2231-999a typea comment-999.0
2232-999a typea comment-999.0
2233-999a typea comment-999.0
2234-999a typea comment-999.0
2235-999a typea comment-999.0
2236-999a typea comment-999.0
2237-999a typea comment-999.0
2238-999a typea comment-999.0
2239-999a typea comment-999.0
2240-999a typea comment-999.0
2241-999a typea comment-999.0
2242-999a typea comment-999.0
2243-999a typea comment-999.0
2244-999a typea comment-999.0
2245-999a typea comment-999.0
2246-999a typea comment-999.0
2247-999a typea comment-999.0
2248-999a typea comment-999.0
2249-999a typea comment-999.0
2250-999a typea comment-999.0
2251-999a typea comment-999.0
2252-999a typea comment-999.0
2253-999a typea comment-999.0
2254-999a typea comment-999.0
2255-999a typea comment-999.0
2256-999a typea comment-999.0
2257-999a typea comment-999.0
2258-999a typea comment-999.0
2259-999a typea comment-999.0
2260-999a typea comment-999.0
2261-999a typea comment-999.0
2262-999a typea comment-999.0
2263-999a typea comment-999.0
2264-999a typea comment-999.0
2265-999a typea comment-999.0
2266-999a typea comment-999.0
2267-999a typea comment-999.0
2268-999a typea comment-999.0
2269-999a typea comment-999.0
2270-999a typea comment-999.0
2271-999a typea comment-999.0
2272-999a typea comment-999.0
2273-999a typea comment-999.0
2274-999a typea comment-999.0
2275-999a typea comment-999.0
2276-999a typea comment-999.0
2277-999a typea comment-999.0
2278-999a typea comment-999.0
2279-999a typea comment-999.0
2280-999a typea comment-999.0
2281-999a typea comment-999.0
2282-999a typea comment-999.0
2283-999a typea comment-999.0
2284-999a typea comment-999.0
2285-999a typea comment-999.0
2286-999a typea comment-999.0
2287-999a typea comment-999.0
2288-999a typea comment-999.0
2289-999a typea comment-999.0
2290-999a typea comment-999.0
2291-999a typea comment-999.0
2292-999a typea comment-999.0
2293-999a typea comment-999.0
2294-999a typea comment-999.0
2295-999a typea comment-999.0
2296-999a typea comment-999.0
2297-999a typea comment-999.0
2298-999a typea comment-999.0
2299-999a typea comment-999.0
2300-999a typea comment-999.0
2301-999a typea comment-999.0
2302-999a typea comment-999.0
2303-999a typea comment-999.0
2304-999a typea comment-999.0
2305-999a typea comment-999.0
2306-999a typea comment-999.0
2307-999a typea comment-999.0
2308-999a typea comment-999.0
2309-999a typea comment-999.0
2310-999a typea comment-999.0
2311-999a typea comment-999.0
2312-999a typea comment-999.0
2313-999a typea comment-999.0
2314-999a typea comment-999.0
2315-999a typea comment-999.0
2316-999a typea comment-999.0
2317-999a typea comment-999.0
2318-999a typea comment-999.0
2319-999a typea comment-999.0
2320-999a typea comment-999.0
2321-999a typea comment-999.0
2322-999a typea comment-999.0
2323-999a typea comment-999.0
2324-999a typea comment-999.0
2325-999a typea comment-999.0
2326-999a typea comment-999.0
2327-999a typea comment-999.0
2328-999a typea comment-999.0
2329-999a typea comment-999.0
2330-999a typea comment-999.0
2331-999a typea comment-999.0
2332-999a typea comment-999.0
2333-999a typea comment-999.0
2334-999a typea comment-999.0
2335-999a typea comment-999.0
2336-999a typea comment-999.0
2337-999a typea comment-999.0
2338-999a typea comment-999.0
2339-999a typea comment-999.0
2340-999a typea comment-999.0
2341-999a typea comment-999.0
2342-999a typea comment-999.0
2343-999a typea comment-999.0
2344-999a typea comment-999.0
2345-999a typea comment-999.0
2346-999a typea comment-999.0
2347-999a typea comment-999.0
2348-999a typea comment-999.0
2349-999a typea comment-999.0
2350-999a typea comment-999.0
2351-999a typea comment-999.0
2352-999a typea comment-999.0
2353-999a typea comment-999.0
2354-999a typea comment-999.0
2355-999a typea comment-999.0
2356-999a typea comment-999.0
2357-999a typea comment-999.0
2358-999a typea comment-999.0
2359-999a typea comment-999.0
2360-999a typea comment-999.0
2361-999a typea comment-999.0
2362-999a typea comment-999.0
2363-999a typea comment-999.0
2364-999a typea comment-999.0
2365-999a typea comment-999.0
2366-999a typea comment-999.0
2367-999a typea comment-999.0
2368-999a typea comment-999.0
2369-999a typea comment-999.0
2370-999a typea comment-999.0
2371-999a typea comment-999.0
2372-999a typea comment-999.0
2373-999a typea comment-999.0
2374-999a typea comment-999.0
2375-999a typea comment-999.0
2376-999a typea comment-999.0
2377-999a typea comment-999.0
2378-999a typea comment-999.0
2379-999a typea comment-999.0
2380-999a typea comment-999.0
2381-999a typea comment-999.0
2382-999a typea comment-999.0
2383-999a typea comment-999.0
2384-999a typea comment-999.0
2385-999a typea comment-999.0
2386-999a typea comment-999.0
2387-999a typea comment-999.0
2388-999a typea comment-999.0
2389-999a typea comment-999.0
2390-999a typea comment-999.0
2391-999a typea comment-999.0
2392-999a typea comment-999.0
2393-999a typea comment-999.0
2394-999a typea comment-999.0
2395-999a typea comment-999.0
2396-999a typea comment-999.0
2397-999a typea comment-999.0
2398-999a typea comment-999.0
2399-999a typea comment-999.0
2400-999a typea comment-999.0
2401-999a typea comment-999.0
2402-999a typea comment-999.0
2403-999a typea comment-999.0
2404-999a typea comment-999.0
2405-999a typea comment-999.0
2406-999a typea comment-999.0
2407-999a typea comment-999.0
2408-999a typea comment-999.0
2409-999a typea comment-999.0
2410-999a typea comment-999.0
2411-999a typea comment-999.0
2412-999a typea comment-999.0
2413-999a typea comment-999.0
2414-999a typea comment-999.0
2415-999a typea comment-999.0
2416-999a typea comment-999.0
2417-999a typea comment-999.0
2418-999a typea comment-999.0
2419-999a typea comment-999.0
2420-999a typea comment-999.0
2421-999a typea comment-999.0
2422-999a typea comment-999.0
2423-999a typea comment-999.0
2424-999a typea comment-999.0
2425-999a typea comment-999.0
2426-999a typea comment-999.0
2427-999a typea comment-999.0
2428-999a typea comment-999.0
2429-999a typea comment-999.0
2430-999a typea comment-999.0
2431-999a typea comment-999.0
2432-999a typea comment-999.0
2433-999a typea comment-999.0
2434-999a typea comment-999.0
2435-999a typea comment-999.0
2436-999a typea comment-999.0
2437-999a typea comment-999.0
2438-999a typea comment-999.0
2439-999a typea comment-999.0
2440-999a typea comment-999.0
2441-999a typea comment-999.0
2442-999a typea comment-999.0
2443-999a typea comment-999.0
2444-999a typea comment-999.0
2445-999a typea comment-999.0
2446-999a typea comment-999.0
2447-999a typea comment-999.0
2448-999a typea comment-999.0
2449-999a typea comment-999.0
2450-999a typea comment-999.0
2451-999a typea comment-999.0
2452-999a typea comment-999.0
2453-999a typea comment-999.0
2454-999a typea comment-999.0
2455-999a typea comment-999.0
2456-999a typea comment-999.0
2457-999a typea comment-999.0
2458-999a typea comment-999.0
2459-999a typea comment-999.0
2460-999a typea comment-999.0
2461-999a typea comment-999.0
2462-999a typea comment-999.0
2463-999a typea comment-999.0
2464-999a typea comment-999.0
2465-999a typea comment-999.0
2466-999a typea comment-999.0
2467-999a typea comment-999.0
2468-999a typea comment-999.0
2469-999a typea comment-999.0
2470-999a typea comment-999.0
2471-999a typea comment-999.0
2472-999a typea comment-999.0
2473-999a typea comment-999.0
2474-999a typea comment-999.0
2475-999a typea comment-999.0
2476-999a typea comment-999.0
2477-999a typea comment-999.0
2478-999a typea comment-999.0
2479-999a typea comment-999.0
2480-999a typea comment-999.0
2481-999a typea comment-999.0
2482-999a typea comment-999.0
2483-999a typea comment-999.0
2484-999a typea comment-999.0
2485-999a typea comment-999.0
2486-999a typea comment-999.0
2487-999a typea comment-999.0
2488-999a typea comment-999.0
2489-999a typea comment-999.0
2490-999a typea comment-999.0
2491-999a typea comment-999.0
2492-999a typea comment-999.0
2493-999a typea comment-999.0
2494-999a typea comment-999.0
2495-999a typea comment-999.0
2496-999a typea comment-999.0
2497-999a typea comment-999.0
2498-999a typea comment-999.0
2499-999a typea comment-999.0
2500-999a typea comment-999.0
2501-999a typea comment-999.0
2502-999a typea comment-999.0
2503-999a typea comment-999.0
2504-999a typea comment-999.0
2505-999a typea comment-999.0
2506-999a typea comment-999.0
2507-999a typea comment-999.0
2508-999a typea comment-999.0
2509-999a typea comment-999.0
2510-999a typea comment-999.0
2511-999a typea comment-999.0
2512-999a typea comment-999.0
2513-999a typea comment-999.0
2514-999a typea comment-999.0
2515-999a typea comment-999.0
2516-999a typea comment-999.0
2517-999a typea comment-999.0
2518-999a typea comment-999.0
2519-999a typea comment-999.0
2520-999a typea comment-999.0
2521-999a typea comment-999.0
2522-999a typea comment-999.0
2523-999a typea comment-999.0
2524-999a typea comment-999.0
2525-999a typea comment-999.0
2526-999a typea comment-999.0
2527-999a typea comment-999.0
2528-999a typea comment-999.0
2529-999a typea comment-999.0
2530-999a typea comment-999.0
2531-999a typea comment-999.0
2532-999a typea comment-999.0
2533-999a typea comment-999.0
2534-999a typea comment-999.0
2535-999a typea comment-999.0
2536-999a typea comment-999.0
2537-999a typea comment-999.0
2538-999a typea comment-999.0
2539-999a typea comment-999.0
2540-999a typea comment-999.0
2541-999a typea comment-999.0
2542-999a typea comment-999.0
2543-999a typea comment-999.0
2544-999a typea comment-999.0
2545-999a typea comment-999.0
2546-999a typea comment-999.0
2547-999a typea comment-999.0
2548-999a typea comment-999.0
2549-999a typea comment-999.0
2550-999a typea comment-999.0
2551-999a typea comment-999.0
2552-999a typea comment-999.0
2553-999a typea comment-999.0
2554-999a typea comment-999.0
2555-999a typea comment-999.0
2556-999a typea comment-999.0
2557-999a typea comment-999.0
2558-999a typea comment-999.0
2559-999a typea comment-999.0
2560-999a typea comment-999.0
2561-999a typea comment-999.0
2562-999a typea comment-999.0
2563-999a typea comment-999.0
2564-999a typea comment-999.0
2565-999a typea comment-999.0
2566-999a typea comment-999.0
2567-999a typea comment-999.0
2568-999a typea comment-999.0
2569-999a typea comment-999.0
2570-999a typea comment-999.0
2571-999a typea comment-999.0
2572-999a typea comment-999.0
2573-999a typea comment-999.0
2574-999a typea comment-999.0
2575-999a typea comment-999.0
2576-999a typea comment-999.0
2577-999a typea comment-999.0
2578-999a typea comment-999.0
2579-999a typea comment-999.0
2580-999a typea comment-999.0
2581-999a typea comment-999.0
2582-999a typea comment-999.0
2583-999a typea comment-999.0
2584-999a typea comment-999.0
2585-999a typea comment-999.0
2586-999a typea comment-999.0
2587-999a typea comment-999.0
2588-999a typea comment-999.0
2589-999a typea comment-999.0
2590-999a typea comment-999.0
2591-999a typea comment-999.0
2592-999a typea comment-999.0
2593-999a typea comment-999.0
2594-999a typea comment-999.0
2595-999a typea comment-999.0
2596-999a typea comment-999.0
2597-999a typea comment-999.0
2598-999a typea comment-999.0
2599-999a typea comment-999.0
2600-999a typea comment-999.0
2601-999a typea comment-999.0
2602-999a typea comment-999.0
2603-999a typea comment-999.0
2604-999a typea comment-999.0
2605-999a typea comment-999.0
2606-999a typea comment-999.0
2607-999a typea comment-999.0
2608-999a typea comment-999.0
2609-999a typea comment-999.0
2610-999a typea comment-999.0
2611-999a typea comment-999.0
2612-999a typea comment-999.0
2613-999a typea comment-999.0
2614-999a typea comment-999.0
2615-999a typea comment-999.0
2616-999a typea comment-999.0
2617-999a typea comment-999.0
2618-999a typea comment-999.0
2619-999a typea comment-999.0
2620-999a typea comment-999.0
2621-999a typea comment-999.0
2622-999a typea comment-999.0
2623-999a typea comment-999.0
2624-999a typea comment-999.0
2625-999a typea comment-999.0
2626-999a typea comment-999.0
2627-999a typea comment-999.0
2628-999a typea comment-999.0
2629-999a typea comment-999.0
2630-999a typea comment-999.0
2631-999a typea comment-999.0
2632-999a typea comment-999.0
2633-999a typea comment-999.0
2634-999a typea comment-999.0
2635-999a typea comment-999.0
2636-999a typea comment-999.0
2637-999a typea comment-999.0
2638-999a typea comment-999.0
2639-999a typea comment-999.0
2640-999a typea comment-999.0
2641-999a typea comment-999.0
2642-999a typea comment-999.0
2643-999a typea comment-999.0
2644-999a typea comment-999.0
2645-999a typea comment-999.0
2646-999a typea comment-999.0
2647-999a typea comment-999.0
2648-999a typea comment-999.0
2649-999a typea comment-999.0
2650-999a typea comment-999.0
2651-999a typea comment-999.0
2652-999a typea comment-999.0
2653-999a typea comment-999.0
2654-999a typea comment-999.0
2655-999a typea comment-999.0
2656-999a typea comment-999.0
2657-999a typea comment-999.0
2658-999a typea comment-999.0
2659-999a typea comment-999.0
2660-999a typea comment-999.0
2661-999a typea comment-999.0
2662-999a typea comment-999.0
2663-999a typea comment-999.0
2664-999a typea comment-999.0
2665-999a typea comment-999.0
2666-999a typea comment-999.0
2667-999a typea comment-999.0
2668-999a typea comment-999.0
2669-999a typea comment-999.0
2670-999a typea comment-999.0
2671-999a typea comment-999.0
2672-999a typea comment-999.0
2673-999a typea comment-999.0
2674-999a typea comment-999.0
2675-999a typea comment-999.0
2676-999a typea comment-999.0
2677-999a typea comment-999.0
2678-999a typea comment-999.0
2679-999a typea comment-999.0
2680-999a typea comment-999.0
2681-999a typea comment-999.0
2682-999a typea comment-999.0
2683-999a typea comment-999.0
2684-999a typea comment-999.0
2685-999a typea comment-999.0
2686-999a typea comment-999.0
2687-999a typea comment-999.0
2688-999a typea comment-999.0
2689-999a typea comment-999.0
2690-999a typea comment-999.0
2691-999a typea comment-999.0
2692-999a typea comment-999.0
2693-999a typea comment-999.0
2694-999a typea comment-999.0
2695-999a typea comment-999.0
2696-999a typea comment-999.0
2697-999a typea comment-999.0
2698-999a typea comment-999.0
2699-999a typea comment-999.0
2700-999a typea comment-999.0
2701-999a typea comment-999.0
2702-999a typea comment-999.0
2703-999a typea comment-999.0
2704-999a typea comment-999.0
2705-999a typea comment-999.0
2706-999a typea comment-999.0
2707-999a typea comment-999.0
2708-999a typea comment-999.0
2709-999a typea comment-999.0
2710-999a typea comment-999.0
2711-999a typea comment-999.0
2712-999a typea comment-999.0
2713-999a typea comment-999.0
2714-999a typea comment-999.0
2715-999a typea comment-999.0
2716-999a typea comment-999.0
2717-999a typea comment-999.0
2718-999a typea comment-999.0
2719-999a typea comment-999.0
2720-999a typea comment-999.0
2721-999a typea comment-999.0
2722-999a typea comment-999.0
2723-999a typea comment-999.0
2724-999a typea comment-999.0
2725-999a typea comment-999.0
2726-999a typea comment-999.0
2727-999a typea comment-999.0
2728-999a typea comment-999.0
2729-999a typea comment-999.0
2730-999a typea comment-999.0
2731-999a typea comment-999.0
2732-999a typea comment-999.0
2733-999a typea comment-999.0
2734-999a typea comment-999.0
2735-999a typea comment-999.0
2736-999a typea comment-999.0
2737-999a typea comment-999.0
2738-999a typea comment-999.0
2739-999a typea comment-999.0
2740-999a typea comment-999.0
2741-999a typea comment-999.0
2742-999a typea comment-999.0
2743-999a typea comment-999.0
2744-999a typea comment-999.0
2745-999a typea comment-999.0
2746-999a typea comment-999.0
2747-999a typea comment-999.0
2748-999a typea comment-999.0
2749-999a typea comment-999.0
2750-999a typea comment-999.0
2751-999a typea comment-999.0
2752-999a typea comment-999.0
2753-999a typea comment-999.0
2754-999a typea comment-999.0
2755-999a typea comment-999.0
2756-999a typea comment-999.0
2757-999a typea comment-999.0
2758-999a typea comment-999.0
2759-999a typea comment-999.0
2760-999a typea comment-999.0
2761-999a typea comment-999.0
2762-999a typea comment-999.0
2763-999a typea comment-999.0
2764-999a typea comment-999.0
2765-999a typea comment-999.0
2766-999a typea comment-999.0
2767-999a typea comment-999.0
2768-999a typea comment-999.0
2769-999a typea comment-999.0
2770-999a typea comment-999.0
2771-999a typea comment-999.0
2772-999a typea comment-999.0
2773-999a typea comment-999.0
2774-999a typea comment-999.0
2775-999a typea comment-999.0
2776-999a typea comment-999.0
2777-999a typea comment-999.0
2778-999a typea comment-999.0
2779-999a typea comment-999.0
2780-999a typea comment-999.0
2781-999a typea comment-999.0
2782-999a typea comment-999.0
2783-999a typea comment-999.0
2784-999a typea comment-999.0
2785-999a typea comment-999.0
2786-999a typea comment-999.0
2787-999a typea comment-999.0
2788-999a typea comment-999.0
2789-999a typea comment-999.0
2790-999a typea comment-999.0
2791-999a typea comment-999.0
2792-999a typea comment-999.0
2793-999a typea comment-999.0
2794-999a typea comment-999.0
2795-999a typea comment-999.0
2796-999a typea comment-999.0
2797-999a typea comment-999.0
2798-999a typea comment-999.0
2799-999a typea comment-999.0
2800-999a typea comment-999.0
2801-999a typea comment-999.0
2802-999a typea comment-999.0
2803-999a typea comment-999.0
2804-999a typea comment-999.0
2805-999a typea comment-999.0
2806-999a typea comment-999.0
2807-999a typea comment-999.0
2808-999a typea comment-999.0
2809-999a typea comment-999.0
2810-999a typea comment-999.0
2811-999a typea comment-999.0
2812-999a typea comment-999.0
2813-999a typea comment-999.0
2814-999a typea comment-999.0
2815-999a typea comment-999.0
2816-999a typea comment-999.0
2817-999a typea comment-999.0
2818-999a typea comment-999.0
2819-999a typea comment-999.0
2820-999a typea comment-999.0
2821-999a typea comment-999.0
2822-999a typea comment-999.0
2823-999a typea comment-999.0
2824-999a typea comment-999.0
2825-999a typea comment-999.0
2826-999a typea comment-999.0
2827-999a typea comment-999.0
2828-999a typea comment-999.0
2829-999a typea comment-999.0
2830-999a typea comment-999.0
2831-999a typea comment-999.0
2832-999a typea comment-999.0
2833-999a typea comment-999.0
2834-999a typea comment-999.0
2835-999a typea comment-999.0
2836-999a typea comment-999.0
2837-999a typea comment-999.0
2838-999a typea comment-999.0
2839-999a typea comment-999.0
2840-999a typea comment-999.0
2841-999a typea comment-999.0
2842-999a typea comment-999.0
2843-999a typea comment-999.0
2844-999a typea comment-999.0
2845-999a typea comment-999.0
2846-999a typea comment-999.0
2847-999a typea comment-999.0
2848-999a typea comment-999.0
2849-999a typea comment-999.0
2850-999a typea comment-999.0
2851-999a typea comment-999.0
2852-999a typea comment-999.0
2853-999a typea comment-999.0
2854-999a typea comment-999.0
2855-999a typea comment-999.0
2856-999a typea comment-999.0
2857-999a typea comment-999.0
2858-999a typea comment-999.0
2859-999a typea comment-999.0
2860-999a typea comment-999.0
2861-999a typea comment-999.0
2862-999a typea comment-999.0
2863-999a typea comment-999.0
2864-999a typea comment-999.0
2865-999a typea comment-999.0
2866-999a typea comment-999.0
2867-999a typea comment-999.0
2868-999a typea comment-999.0
2869-999a typea comment-999.0
2870-999a typea comment-999.0
2871-999a typea comment-999.0
2872-999a typea comment-999.0
2873-999a typea comment-999.0
2874-999a typea comment-999.0
2875-999a typea comment-999.0
2876-999a typea comment-999.0
2877-999a typea comment-999.0
2878-999a typea comment-999.0
2879-999a typea comment-999.0
2880-999a typea comment-999.0
2881-999a typea comment-999.0
2882-999a typea comment-999.0
2883-999a typea comment-999.0
2884-999a typea comment-999.0
2885-999a typea comment-999.0
2886-999a typea comment-999.0
2887-999a typea comment-999.0
2888-999a typea comment-999.0
2889-999a typea comment-999.0
2890-999a typea comment-999.0
2891-999a typea comment-999.0
2892-999a typea comment-999.0
2893-999a typea comment-999.0
2894-999a typea comment-999.0
2895-999a typea comment-999.0
2896-999a typea comment-999.0
2897-999a typea comment-999.0
2898-999a typea comment-999.0
2899-999a typea comment-999.0
2900-999a typea comment-999.0
2901-999a typea comment-999.0
2902-999a typea comment-999.0
2903-999a typea comment-999.0
2904-999a typea comment-999.0
2905-999a typea comment-999.0
2906-999a typea comment-999.0
2907-999a typea comment-999.0
2908-999a typea comment-999.0
2909-999a typea comment-999.0
2910-999a typea comment-999.0
2911-999a typea comment-999.0
2912-999a typea comment-999.0
2913-999a typea comment-999.0
2914-999a typea comment-999.0
2915-999a typea comment-999.0
2916-999a typea comment-999.0
2917-999a typea comment-999.0
2918-999a typea comment-999.0
2919-999a typea comment-999.0
2920-999a typea comment-999.0
2921-999a typea comment-999.0
2922-999a typea comment-999.0
2923-999a typea comment-999.0
2924-999a typea comment-999.0
2925-999a typea comment-999.0
2926-999a typea comment-999.0
2927-999a typea comment-999.0
2928-999a typea comment-999.0
2929-999a typea comment-999.0
2930-999a typea comment-999.0
2931-999a typea comment-999.0
2932-999a typea comment-999.0
2933-999a typea comment-999.0
2934-999a typea comment-999.0
2935-999a typea comment-999.0
2936-999a typea comment-999.0
2937-999a typea comment-999.0
2938-999a typea comment-999.0
2939-999a typea comment-999.0
2940-999a typea comment-999.0
2941-999a typea comment-999.0
2942-999a typea comment-999.0
2943-999a typea comment-999.0
2944-999a typea comment-999.0
2945-999a typea comment-999.0
2946-999a typea comment-999.0
2947-999a typea comment-999.0
2948-999a typea comment-999.0
2949-999a typea comment-999.0
2950-999a typea comment-999.0
2951-999a typea comment-999.0
2952-999a typea comment-999.0
2953-999a typea comment-999.0
2954-999a typea comment-999.0
2955-999a typea comment-999.0
2956-999a typea comment-999.0
2957-999a typea comment-999.0
2958-999a typea comment-999.0
2959-999a typea comment-999.0
2960-999a typea comment-999.0
2961-999a typea comment-999.0
2962-999a typea comment-999.0
2963-999a typea comment-999.0
2964-999a typea comment-999.0
2965-999a typea comment-999.0
2966-999a typea comment-999.0
2967-999a typea comment-999.0
2968-999a typea comment-999.0
2969-999a typea comment-999.0
2970-999a typea comment-999.0
2971-999a typea comment-999.0
2972-999a typea comment-999.0
2973-999a typea comment-999.0
2974-999a typea comment-999.0
2975-999a typea comment-999.0
2976-999a typea comment-999.0
2977-999a typea comment-999.0
2978-999a typea comment-999.0
2979-999a typea comment-999.0
2980-999a typea comment-999.0
2981-999a typea comment-999.0
2982-999a typea comment-999.0
2983-999a typea comment-999.0
2984-999a typea comment-999.0
2985-999a typea comment-999.0
2986-999a typea comment-999.0
2987-999a typea comment-999.0
2988-999a typea comment-999.0
2989-999a typea comment-999.0
2990-999a typea comment-999.0
2991-999a typea comment-999.0
2992-999a typea comment-999.0
2993-999a typea comment-999.0
2994-999a typea comment-999.0
2995-999a typea comment-999.0
2996-999a typea comment-999.0
2997-999a typea comment-999.0
2998-999a typea comment-999.0
2999-999a typea comment-999.0
3000-999a typea comment-999.0
3001-999a typea comment-999.0
3002-999a typea comment-999.0
3003-999a typea comment-999.0
3004-999a typea comment-999.0
3005-999a typea comment-999.0
3006-999a typea comment-999.0
3007-999a typea comment-999.0
3008-999a typea comment-999.0
3009-999a typea comment-999.0
3010-999a typea comment-999.0
3011-999a typea comment-999.0
3012-999a typea comment-999.0
3013-999a typea comment-999.0
3014-999a typea comment-999.0
3015-999a typea comment-999.0
3016-999a typea comment-999.0
3017-999a typea comment-999.0
3018-999a typea comment-999.0
3019-999a typea comment-999.0
3020-999a typea comment-999.0
3021-999a typea comment-999.0
3022-999a typea comment-999.0
3023-999a typea comment-999.0
3024-999a typea comment-999.0
3025-999a typea comment-999.0
3026-999a typea comment-999.0
3027-999a typea comment-999.0
3028-999a typea comment-999.0
3029-999a typea comment-999.0
3030-999a typea comment-999.0
3031-999a typea comment-999.0
3032-999a typea comment-999.0
3033-999a typea comment-999.0
3034-999a typea comment-999.0
3035-999a typea comment-999.0
3036-999a typea comment-999.0
3037-999a typea comment-999.0
3038-999a typea comment-999.0
3039-999a typea comment-999.0
3040-999a typea comment-999.0
3041-999a typea comment-999.0
3042-999a typea comment-999.0
3043-999a typea comment-999.0
3044-999a typea comment-999.0
3045-999a typea comment-999.0
3046-999a typea comment-999.0
3047-999a typea comment-999.0
3048-999a typea comment-999.0
3049-999a typea comment-999.0
3050-999a typea comment-999.0
3051-999a typea comment-999.0
3052-999a typea comment-999.0
3053-999a typea comment-999.0
3054-999a typea comment-999.0
3055-999a typea comment-999.0
3056-999a typea comment-999.0
3057-999a typea comment-999.0
3058-999a typea comment-999.0
3059-999a typea comment-999.0
3060-999a typea comment-999.0
3061-999a typea comment-999.0
3062-999a typea comment-999.0
3063-999a typea comment-999.0
3064-999a typea comment-999.0
3065-999a typea comment-999.0
3066-999a typea comment-999.0
3067-999a typea comment-999.0
3068-999a typea comment-999.0
3069-999a typea comment-999.0
3070-999a typea comment-999.0
3071-999a typea comment-999.0
3072-999a typea comment-999.0
3073-999a typea comment-999.0