Index: trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh =================================================================== --- trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42415) +++ trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42416) @@ -85,4 +85,9 @@ if ($mystate == "md5chk_s2.fail") goto good_state + # stage 3: + if ($mystate == "fixchip_s3.new") goto good_state + if ($mystate == "fixchip_s3.done") goto good_state + if ($mystate == "fixchip_s3.fail") goto good_state + # disable the rsyncs stages: run in directory from b-node # if ($mystate == "rsync_s1.new") goto good_state Index: trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage3.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage3.pl (revision 42416) +++ trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage3.pl (revision 42416) @@ -0,0 +1,285 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations_stage3.pl --dateobs (night) --chiplist (file) + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ('nebulous.config'); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); +my %volarray = &get_neb_volumes; + +# choose ITC machines on which to place backup copies: +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$dateword/$chip_id.uris.txt"; + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these were replicated in stage 2, so actually match one of the CUL1 or CUL2 entries + # if ($state eq "DUPS:") { $Ndups++; next; } + # if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + # if ($state eq "CUL1:") { $Ncul1++; next; } + # if ($state eq "CUL2:") { $Ncul2++; next; } + # if ($state eq "CUL3:") { $Ncul3++; next; } + + # one copy at ITC: neb-replicate will create a copy at ATRC + if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:") || ($state eq "DUPS:") || ($state eq "XBCK:")) { + + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($ext_id); + + my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id); + + if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n"; next; } + if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; } + + my $NcullTotal = @$cull_list_r; + foreach my $instance (@$cull_list_r) { + my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|; + unless (defined $volarray{$volname}) { die "ERROR: invalid volume name $volname\n"; } + # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n"; + my $status = &vsystem ("neb-cull --volume $volname $ext_id"); + if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; } + } + # print STDERR "Ncull Total $NcullTotal\n"; + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_chip_instances { + + my $ext_id = $_[0]; + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# select 1 instance with xattr = 0, 1 with xattr = 1. +# the rest should be culled. + +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $main_keep = ""; + my $back_keep = ""; + my @cull_list = (); + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + # the number of returned values per row must match the query above (in get_chip_instances) + my $Nvalue = @values; + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; } + if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; } + + # if we make it here, we already have an instance at this location + push (@cull_list, $instance); + } + return ($main_keep, $back_keep, \@cull_list); +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +# +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my %volarray; + + my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + $volarray{$values[0]} = 1; + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return %volarray; +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} Index: trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s3.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s3.pt (revision 42416) +++ trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s3.pt (revision 42416) @@ -0,0 +1,119 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_S3_INIT)) + book init fixchip_s3.book + $RAWFIX_FIXCHIP_S3_INIT = 1 +end + +# select nights in state fixchip_s3.new +# output: fixchip_s3.book +task nights.fixchip_s3.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip_s3.load.stdout.log + stderr $LOGDIR/fixchip_s3.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip_s3.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s3.book + book delpage fixchip_s3.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip_s3.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/fixchip_s3.run.stdout.log + stderr $LOGDIR/fixchip_s3.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip_s3.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip_s3.book (pantaskState == INIT) + book getpage fixchip_s3.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip_s3.book $pageName pantaskState RUN + book getword fixchip_s3.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command fix_chip_locations_stage3.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword fixchip_s3.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip_s3.done + book delpage fixchip_s3.book $options:0 + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip_s3.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip_s3.fail + end + + task.exit crash + showcommand crash + book setword fixchip_s3.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip_s3.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip_s3.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip_s3.fail + end +end + Index: trunk/tools/eam/rawfix.20230221/src/rawfix.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.pt (revision 42415) +++ trunk/tools/eam/rawfix.20230221/src/rawfix.pt (revision 42416) @@ -28,6 +28,7 @@ # input rawfix.md5chk.pt # input rawfix.fixchip_s2.pt -input rawfix.md5sum_s2.pt -#input rawfix.md5chk_s2.pt +# input rawfix.md5sum_s2.pt +# input rawfix.md5chk_s2.pt +# input rawfix.fixchip_s3.pt # no need to run rsync if we use the b-node working directory: