Index: /trunk/tools/stsci_disks/p1_clear.pl =================================================================== --- /trunk/tools/stsci_disks/p1_clear.pl (revision 38279) +++ /trunk/tools/stsci_disks/p1_clear.pl (revision 38279) @@ -0,0 +1,298 @@ +#! /usr/bin/env perl + +use DBI; +use PS::IPP::Config 1.01 qw( :standard ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use IPC::Cmd 0.36 qw( can_run run ); +use Carp; +use Pod::Usage qw( pod2usage ); + +use Nebulous::Server::SQL; +use Nebulous::Client; +use SQL::Interp qw( sql_interp ); +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use File::Basename; + + +my ($host,$volume,$dir); +GetOptions( + 'host=s' => \$host, + 'volume=s' => \$volume, + 'dir=s' => \$dir) || pod2usage(2 ); + +my ( $db, $dbhost, $dbpass, $dbuser); +my $conffile = "/home/panstarrs/ipp/ippconfig/nebulous.config"; +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_DB\s+/) { + $db = (split /\s+/,$_)[2]; + } + elsif ($_ =~ /NEB_DBHOST/) { + $dbhost = (split /\s+/,$_)[2]; + } + elsif ($_ =~ /NEB_USER/) { + $dbuser = (split /\s+/,$_)[2]; + } + elsif ($_ =~ /NEB_PASS/) { + $dbpass = (split /\s+/,$_)[2]; + } + } + close(IN); +# print STDERR "$db $dbhost $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $host && defined $volume && defined $dir && + defined $db && defined $dbhost && defined $dbuser && defined $dbpass; + +my $dbh = DBI->connect( + "DBI:mysql:database=$db:host=$dbhost", + $dbuser, + $dbpass, + { + RaiseError => 1, + PrintError => 0, + AutoCommit => 0, + }, + ); +my $sql = Nebulous::Server::SQL->new(); + +my $server = $ENV{'NEB_SERVER'}; +my $neb = Nebulous::Client->new( + proxy => "$server", + ); + +my $time = time(); +my $logDest = "neb://any/stsci_shuffle/${host}.${volume}/${dir}.${time}"; +my $ipprc = PS::IPP::Config->new( "GPC1" ) or die "Could not create config object.\n"; +$ipprc->redirect_output($logDest) or die "Could not redirect output to logfile ${logDest}\n"; + +my $unknown_dir = "/data/ippb06.0/stsci_unknown/"; +my $root = "/data/${host}.${volume}/nebulous/${dir}"; +my @subdirectories = <$root/*>; + +#print "/data/${host}.${volume}/nebulous/${dir}/*\n"; +#print "$#subdirectories\n"; +foreach my $subdir (@subdirectories) { + my @files = <${subdir}/*>; + my $otime = $time; + $time = time(); + my $dtime = $time - $otime; + print "LOG BEGIN $subdir $time $dtime\n"; + + my @belongs; + my @keys; + my @vols; + my @md5sums; + +# Pre scan + for (my $i = 0; $i <= $#files; $i++) { + my $f = $files[$i]; + my ($ins_id, $belongs); + $ins_id = $f; + $ins_id =~ s/^$subdir\///; + $ins_id =~ s/\..*//; + + $belongs[$i] = 0; + + if ($ins_id > 4956576912) { + if ($f =~ /LAP.PV3.20140730/) { # Only PV3 + if (($f =~ /wrp/)) { + if (($f =~ /fits/)||($f =~ /cmf/)||($f =~ /psf/)) { + $belongs[$i] = 1; + } + } + if (($f =~ /stk/)&&($f =~ /fits/)&&($f =~ /unconv/)) { + $belongs[$i] = 1; + } + } + } + + if ($belongs[$i] == 0) { + $keys[$i] = $f; + $keys[$i] =~ s/.*\/${ins_id}\.//; + $keys[$i] =~ s%:%/%g; + # Compressed fits + if ($keys[$i] =~ /.fits.fz$/) { + $keys[$i] =~ s/.fits.fz$/.fits/; + } + + $md5sums[$i] = local_md5sum($f); + # Get a target + $vols[$i] = get_volume($keys[$i]); + if (defined($vols[$i])) { + $vols[$i] = $vols[$i] . ".0"; + } + } + elsif ($belongs[$i] == 1) { + $md5sums[$i] = 'NOT_CALCULATED'; + $vols[$i] = 'ippXXX.Y'; + $keys[$i] = 'NO_REPLICATE'; + } + if (defined($vols[$i])) { + print "## $ins_id $md5sums[$i] $vols[$i] $belongs[$i] $f $keys[$i] \n"; + } + else { + print "## $ins_id $md5sums[$i] NOVOL_BECAUSE_UNKNOWN $belongs[$i] $f $keys[$i] \n"; + } + } + $otime = $time; + $time = time(); + $dtime = $time - $otime; + print "LOG SCAN $subdir $time $dtime\n"; +# die; +# Do a replication pass + for (my $i = 0; $i <= $#files; $i++) { + if ($belongs[$i] == 0) { + if (defined($vols[$i])) { + $neb->replicate($keys[$i],$vols[$i]) or die "failed to replicate a copy to the backup node"; + if ($@) { die $@; } + } + else { + # Key wasn't defiend + my $dir = dirname($keys[$i]); + system("mkdir -p ${unknown_dir}/${dir}"); + system("cp -a $files[$i] ${unknown_dir}/$keys[$i]"); + } + system("sync") == 0 or die "Couldn't sync?"; + } + } + $otime = $time; + $time = time(); + $dtime = $time - $otime; + print "LOG REPL $subdir $time $dtime\n"; + +# Do the check pass + for (my $i = 0; $i <= $#files; $i++) { + my $uri; + if ($belongs[$i] == 0) { + if (defined($vols[$i])) { + my $uris = $neb->find_instances($keys[$i],$vols[$i]); + @$uris = map {URI->new($_)->file if $_} @$uris; + $uri = ${ $uris }[0]; + } + else { + $uri = "${unknown_dir}/$keys[$i]"; + } + + my $tmpmd5 = local_md5sum($uri); + if ($tmpmd5 ne $md5sums[$i]) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sums[$i]"; + } + # End my best validation thought. + + if (defined($vols[$i])) { + $neb->cull($keys[$i],$host . "." . $volume); + } + else { + system("rm $files[$i]"); + } + } + } + + $otime = $time; + $time = time(); + $dtime = $time - $otime; + print "LOG END $subdir $time $dtime\n"; + die; +} + +sub get_volume { + my $key = shift; + + + my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet); + my $max_used_space = 0.98; + my $topfew_count = 15; + eval { + my $rows; + + my $query = $dbh->prepare( $sql->get_cabinets_for_ext_id ); + $rows = $query->execute($key); + unless ($rows > 0) { + $query->finish; + die("Requested key $key does not exist"); + } + if ($rows == 1) { + ($forbidden_cabinet) = $query->fetchrow_array; + unless (defined($forbidden_cabinet)) { + $forbidden_cabinet = 0; + } + $query->finish; + } + else { + $forbidden_cabinet = 0; + $query->finish; + } + + + $query = $dbh->prepare( $sql->get_replication_volume_for_ext_id ); + # ext_id, %free, avaiable, allocate + $rows = $query->execute($key, $max_used_space, 1, 1, $forbidden_cabinet, $topfew_count); + # XXX destinguish between non-existant and unaviable + unless ($rows > 0) { + $query->finish; + die("can't find a suitable storage volume to replicate $key to"); + } + # when matching by name we shouldn't ever match more than once + if ($rows > 1) { + $query->finish; + die("affected row count is $rows instead of 1"); + } + + my $free; + ($vol_id, $vol_host, $vol_path, $xattr, $free) = $query->fetchrow_array; +# print "$vol_id $vol_host $vol_path\n"; + $query->finish; + }; + + return($vol_host); +} + + +sub secure_replicull { + my $key = shift; + my $md5sum = shift; + my $rep_vol = shift; + my $del_vol = shift; + + $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + sleep(2); + $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + + $neb->cull($key,$del_vol); +} + +sub local_md5sum { + my $filename = shift; + my $volume = (split /\//, $filename)[2]; + my $host = $volume; + $host =~ s/\.\d//; +# print "$filename $host $volume\n"; + my $response = `ssh $host md5sum $filename`; + chomp($response); + my ($sum, undef) = split /\s+/, $response; + unless(defined($sum)) { + die("Failed to calculate md5sum locally. $filename $host $volume"); + } + return($sum); +} Index: /trunk/tools/stsci_disks/p2_check.pl =================================================================== --- /trunk/tools/stsci_disks/p2_check.pl (revision 38279) +++ /trunk/tools/stsci_disks/p2_check.pl (revision 38279) @@ -0,0 +1,361 @@ +#! /usr/bin/env perl + +use DBI; +use Nebulous::Client; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use Carp; +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use File::Basename; + + +# Set up all the standard things +my $missing_tools; +my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); +my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); +my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); + +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +my ($stage,$stage_id,$dbname); +GetOptions( + 'stage=s' => \$stage, + 'stage_id=s' => \$stage_id, + 'dbname=s' => \$dbname, +) || pod2usage(2 ); + +my $camera = 'GPC1'; +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set_up", $stage, $stage_id); +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + + +# Define which things we want to look at, and where and how many of each. +my %components = (); +my $command = ''; +if ($stage eq 'warp') { + %components = ('PSWARP.OUTPUT' => { 'local' => 0, 'stsci' => 1 }, + 'PSWARP.OUTPUT.MASK' => { 'local' => 0, 'stsci' => 1 }, + 'PSWARP.OUTPUT.VARIANCE' => { 'local' => 0, 'stsci' => 1 }, + 'PSWARP.OUTPUT.SOURCES' => { 'local' => 2, 'stsci' => 1 }, + 'PSWARP.CONFIG' => { 'local' => 2, 'stsci' => 1} + ); + $command = "$warptool -warped -warp_id $stage_id -dbname $dbname"; +} +elsif ($stage eq 'stack') { + %components = ('PPSTACK.UNCONV' => { 'local' => 2, 'stsci' => 1 }, + 'PPSTACK.UNCONV.MASK' => { 'local' => 2, 'stsci' => 1 }, + 'PPSTACK.UNCONV.VARIANCE' => { 'local' => 2, 'stsci' => 1 }, + 'PPSTACK.UNCONV.EXP' => { 'local' => 2, 'stsci' => 1 }, + 'PPSTACK.UNCONV.EXPNUM' => { 'local' => 2, 'stsci' => 1 }, + 'PPSTACK.UNCONV.EXPWT' => { 'local' => 2, 'stsci' => 1 }, + 'PSPHOT.PSF.SKY.SAVE' => { 'local' => 2, 'stsci' => 1 }, + 'PSPHOT.OUT.CMF.MEF' => { 'local' => 2, 'stsci' => 1 }, + 'PPSTACK.CONFIG' => { 'local' => 2, 'stsci' => 1} + + ); + $command = "$stacktool -sumskyfile -stack_id $stage_id -dbname $dbname"; +} +elsif ($stage eq 'diff') { + %components = ('PPSUB.OUTPUT.KERNELS' => { 'local' => 2, 'stsci' => 1 }, + 'PPSUB.OUTPUT.SOURCES' => { 'local' => 2, 'stsci' => 1 }, + 'PPSUB.CONFIG' => { 'local' => 2, 'stsci' => 1 } + ); + $command = "$difftool -diffskyfile -diff_id $stage_id -dbname $dbname"; +} + +# Get the stage information +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id); +} +my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to run STAGEtool -liststagestuff ", $stage,$stage_id); +my $stageData = parse_md_list($MDlist); + +# Determine a file list; +my @keylist = (); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; +print "LOG BEGIN $stage $stage_id $time $dtime\n"; + +foreach my $stageEntry ( @{ $stageData } ) { + my $quality = $stageEntry->{quality}; + if ($quality != 0) { next; } # SKip things with bad quality. + + my $path_base = $stageEntry->{path_base}; + + foreach my $component (keys %components) { + my $key = $ipprc->filename($component,$path_base); + + # Stealing most of neb-stat here + my %md5sum_uniq; + my $instances = $neb->find_instances($key, 'any'); + my @files = map {URI->new($_)->file if $_} @$instances; + my @stsci_copies = (); + my @local_copies = (); + my @existance; + my @md5sums; + my $local_count = 0; + my $stsci_count = 0; + + for (my $i = 0; $i <= $#files; $i++) { + my $volume; + if (-e $files[$i]) { + $existance[$i] = 1; + ($md5sums[$i],$volume) = local_md5sum($files[$i]); + $md5sum_uniq{$md5sums[$i]} = 1; + if ($volume =~ /stsci/) { + $stsci_count++; + $stsci_copies[$i] = 1; + $local_copies[$i] = 0; + } + elsif ($volume =~ /ipp/) { + $local_count++; + $stsci_copies[$i] = 0; + $local_copies[$i] = 1; + } + else { &my_die("You can't exist somewhere not ipp and not stsci",$stage,$stage_id,$files[$i]); } + + } + } + + # Do the logic at what we found + + + + + + + + + +my $server = $ENV{'NEB_SERVER'}; +my $neb = Nebulous::Client->new( + proxy => "$server", + ); + + +foreach my $subdir (@subdirectories) { + my @files = <${subdir}/*>; + + my @belongs; + my @keys; + my @vols; + my @md5sums; + +# Pre scan + for (my $i = 0; $i <= $#files; $i++) { + my $f = $files[$i]; + my ($ins_id, $belongs); + $ins_id = $f; + $ins_id =~ s/^$subdir\///; + $ins_id =~ s/\..*//; + + $belongs[$i] = 0; + + if ($ins_id > 4956576912) { + if ($f =~ /LAP.PV3.20140730/) { # Only PV3 + if (($f =~ /wrp/)) { + if (($f =~ /fits/)||($f =~ /cmf/)||($f =~ /psf/)) { + $belongs[$i] = 1; + } + } + if (($f =~ /stk/)&&($f =~ /fits/)&&($f =~ /unconv/)) { + $belongs[$i] = 1; + } + } + } + + if ($belongs[$i] == 0) { + $keys[$i] = $f; + $keys[$i] =~ s/.*\/${ins_id}\.//; + $keys[$i] =~ s%:%/%g; + # Compressed fits + if ($keys[$i] =~ /.fits.fz$/) { + $keys[$i] =~ s/.fits.fz$/.fits/; + } + + $md5sums[$i] = local_md5sum($f); + # Get a target + $vols[$i] = get_volume($keys[$i]); + if (defined($vols[$i])) { + $vols[$i] = $vols[$i] . ".0"; + } + } + elsif ($belongs[$i] == 1) { + $md5sums[$i] = 'NOT_CALCULATED'; + $vols[$i] = 'ippXXX.Y'; + $keys[$i] = 'NO_REPLICATE'; + } + if (defined($vols[$i])) { + print "## $ins_id $md5sums[$i] $vols[$i] $belongs[$i] $f $keys[$i] \n"; + } + else { + print "## $ins_id $md5sums[$i] NOVOL_BECAUSE_UNKNOWN $belongs[$i] $f $keys[$i] \n"; + } + } + $otime = $time; + $time = time(); + $dtime = $time - $otime; + print "LOG SCAN $subdir $time $dtime\n"; +# die; +# Do a replication pass + for (my $i = 0; $i <= $#files; $i++) { + if ($belongs[$i] == 0) { + if (defined($vols[$i])) { + $neb->replicate($keys[$i],$vols[$i]) or die "failed to replicate a copy to the backup node"; + if ($@) { die $@; } + } + else { + # Key wasn't defiend + my $dir = dirname($keys[$i]); + system("mkdir -p ${unknown_dir}/${dir}"); + system("cp -a $files[$i] ${unknown_dir}/$keys[$i]"); + } + system("sync") == 0 or die "Couldn't sync?"; + } + } + $otime = $time; + $time = time(); + $dtime = $time - $otime; + print "LOG REPL $subdir $time $dtime\n"; + +# Do the check pass + for (my $i = 0; $i <= $#files; $i++) { + my $uri; + if ($belongs[$i] == 0) { + if (defined($vols[$i])) { + my $uris = $neb->find_instances($keys[$i],$vols[$i]); + @$uris = map {URI->new($_)->file if $_} @$uris; + $uri = ${ $uris }[0]; + } + else { + $uri = "${unknown_dir}/$keys[$i]"; + } + + my $tmpmd5 = local_md5sum($uri); + if ($tmpmd5 ne $md5sums[$i]) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sums[$i]"; + } + # End my best validation thought. + + if (defined($vols[$i])) { + $neb->cull($keys[$i],$host . "." . $volume); + } + else { + system("rm $files[$i]"); + } + } + } + + $otime = $time; + $time = time(); + $dtime = $time - $otime; + print "LOG END $subdir $time $dtime\n"; + die; +} + +sub get_volume { + my $key = shift; + + + my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet); + my $max_used_space = 0.98; + my $topfew_count = 15; + eval { + my $rows; + + my $query = $dbh->prepare( $sql->get_cabinets_for_ext_id ); + $rows = $query->execute($key); + unless ($rows > 0) { + $query->finish; + die("Requested key $key does not exist"); + } + if ($rows == 1) { + ($forbidden_cabinet) = $query->fetchrow_array; + unless (defined($forbidden_cabinet)) { + $forbidden_cabinet = 0; + } + $query->finish; + } + else { + $forbidden_cabinet = 0; + $query->finish; + } + + + $query = $dbh->prepare( $sql->get_replication_volume_for_ext_id ); + # ext_id, %free, avaiable, allocate + $rows = $query->execute($key, $max_used_space, 1, 1, $forbidden_cabinet, $topfew_count); + # XXX destinguish between non-existant and unaviable + unless ($rows > 0) { + $query->finish; + die("can't find a suitable storage volume to replicate $key to"); + } + # when matching by name we shouldn't ever match more than once + if ($rows > 1) { + $query->finish; + die("affected row count is $rows instead of 1"); + } + + my $free; + ($vol_id, $vol_host, $vol_path, $xattr, $free) = $query->fetchrow_array; +# print "$vol_id $vol_host $vol_path\n"; + $query->finish; + }; + + return($vol_host); +} + + +sub secure_replicull { + my $key = shift; + my $md5sum = shift; + my $rep_vol = shift; + my $del_vol = shift; + + $neb->replicate($key,$rep_vol) or die "failed to replicate a copy to the backup node"; + if ($@) { die $@; } + + # Begin my best validation thought + system("sync") == 0 or die "Couldn't sync?"; + my $uris = $neb->find_instances($key,$rep_vol); + @$uris = map {URI->new($_)->file if $_} @$uris; + my $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + sleep(2); + $tmpmd5 = local_md5sum(${ $uris }[0]); + if ($tmpmd5 ne $md5sum) { + die "Post-replication md5sum does not match! $tmpmd5 != $md5sum"; + } + } + # End my best validation thought. + + $neb->cull($key,$del_vol); +} + +sub local_md5sum { + my $filename = shift; + my $volume = (split /\//, $filename)[2]; + my $host = $volume; + $host =~ s/\.\d//; +# print "$filename $host $volume\n"; + my $response = `ssh $host md5sum $filename`; + chomp($response); + my ($sum, undef) = split /\s+/, $response; + unless(defined($sum)) { + die("Failed to calculate md5sum locally. $filename $host $volume"); + } + return($sum,$volume); +}