Index: trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42443) +++ trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42444) @@ -13,6 +13,6 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) -# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage v0 +# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) +# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage 0 my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; @@ -31,10 +31,10 @@ # output files: # instance lists by host: YYYYMMDD/HOSTNAME.inst.STAGE.txt -# chip states by chip: YYYYMMDD/CHIPNAME.uris.txt +# chip states by chip: YYYYMMDD/CHIPNAME.uris.STAGE.txt foreach my $chip_id (@chip_ids) { my $outfh; - open ($outfh, ">$topdir/$dateword/$chip_id.uris.txt"); + open ($outfh, ">$topdir/$dateword/$chip_id.uris.$stage.txt"); # get the list of chip neb paths for one chip @@ -182,5 +182,5 @@ my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); - + return ($rowref, $ext_id); } @@ -338,4 +338,8 @@ pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + # check that stage is an integer + unless (&is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -405,5 +409,8 @@ } - +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + # example of using the return value of arrayref: Index: trunk/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl (revision 42443) +++ (revision ) @@ -1,423 +1,0 @@ -#! /usr/bin/env perl - -# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases - -# TESTRUN can be 1, 2, 3, 4 to get further in the tests -my $TESTRUN = 2; -my $DEBUG = 3; - -use DBI; -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -my ($tgthost,$tgtvol,$subdir,$movelist); - -# USAGE: check_chip_locations.pl --dateobs (night) --chip_id (chip_id) -# USAGE: check_chip_locations.pl --dateobs 2010.05.01 --chip_id XY33 - -my ($dateobs, $chip_id); -GetOptions( 'dateobs=s' => \$dateobs, 'chip_id=s' => \$chip_id) || pod2usage(2); - -my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); -my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); -my $conffile = 'nebulous.config'; - -# parse the nebulous.config file: -if (-e $conffile) { - open(IN,$conffile); - while() { - chomp; - if ($_ =~ /^\s*#/) { next; } # skip lines with leading # - if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } - - if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } - } - close(IN); - print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; - print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; -} - -pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; -pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; -pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; -pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; -pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; - -pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; -pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; -pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; -pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; -pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; - -pod2usage( -msg => "Cannot determine target date", -exitval => 2) unless defined $dateobs; -pod2usage( -msg => "Cannot determine target chip", -exitval => 2) unless defined $chip_id; - -# check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) - -my ($date_year, $date_month, $date_day) = split "/",$dateobs; -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); - -my $time = time(); - -my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; -if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } -my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - -my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; -if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } -my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - -# get the list of chip neb paths: -my @nebarray; -if (1) { - my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$date_year/$date_month/$date_day,00:00:00' and dateobs < '$date_year/$date_month/$date_day,23:59:59'"; - if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } - - my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); - my @tmparray = @$rowref; - my $Nrows = @tmparray; - - if ($DEBUG > 4) { print "found $Nrows chip images\n"; } - - foreach my $valref (@tmparray) { - my @values = @$valref; - # there should only be a single value (only uri selected above) - my $Nvalues = @values; - if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } - if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } - - push (@nebarray, $values[0]); - if ($DEBUG > 4) { print "$values[0]\n"; } - } -} - -# find the neb instances for each chip nebulous entry - -my %hostcount; - -foreach my $nebname (@nebarray) { - - # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits - # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits - # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) - my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; - if ($DEBUG > 5) { print "nebname: $ext_id\n"; } - - my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; - my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); - my @tmparray = @$rowref; - my $Nrows = @tmparray; - - if ($DEBUG > 5) { print "found $Nrows instances\n"; } - - if (0) { - foreach my $valref (@tmparray) { - my @values = @$valref; - # number of values should match list above (can this even fail?) - my $Nvalues = @values; - if ($Nvalues != 5) { print "ERROR: wrong number of fields?\n"; next; } - - if ($DEBUG > 2) { print "uri: $values[0], host: $values[1], avail: $values[2], xattr: $values[3], cab: $values[4]\n"; } - } - } - - # my $summary = "$ext_id $so_id"; - my $summary = "$ext_id"; - - # get user_copies for this storage_object - my $Nmain = 0; - my $Nback = 0; - my $Ndead = 0; - - # xattr = 0 means instance @ ITC (main) - # xattr = 1 means instance @ ATRC (back) - ## how many xattr = 0 and how many xattr = 1 - for (my $j = 0; $j < $Nrows; $j ++) { - my $valref = $tmparray[$j]; - my @values = @$valref; - - my $Nvalue = @values; - - if ($Nvalue != 5) { - print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; - next; - } - my $xattr = $values[3]; - my $hostname = $values[1]; - - # count the number of each host by incrementing the hash element corresponding to the hostname: - $hostcount{$hostname} ++; - - if ($xattr == 0) { $Nmain ++; } - if ($xattr == 1) { $Nback ++; } - if ($xattr > 1) { $Ndead ++; } - } - - # SKIP : already fixed - if (($Nmain == 1) && ($Nback == 1)) { - print STDOUT "GOOD: $summary\n"; - next; - } - - # single copy @ ITC, no backup - # make a backup - if (($Nmain == 1) && ($Nback == 0)) { - print STDOUT "SING: $summary\n"; - my $status = vsystem ("neb-replicate $ext_id"); - if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } - next; - } - - # ITC files with a single backup copies need to be culled - # cull Nmain - 1 - if (($Nmain > 1) && ($Nback == 1)) { - my $Ncull = $Nmain - 1; - print STDOUT "CULL: $summary\n"; - for (my $j = 0; $j < $Ncull; $j ++) { - my $status = vsystem("neb-cull $ext_id"); - if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } - } - next; - } - - # multiple copies @ ITC - # make a backup and cull Nmain - 1 - if (($Nmain > 1) && ($Nback == 0)) { - my $Ncull = $Nmain - 1; - print STDOUT "DUPS: $summary\n"; - my $status = vsystem("neb-replicate $ext_id"); - if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } - for (my $j = 0; $j < $Ncull; $j ++) { - $status = vsystem("neb-cull $ext_id"); - if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } - } - next; - } - - if (($Nmain == 0) && ($Nback == 0) && $Ndead) { - print STDOUT "DEAD: $summary\n"; - next; - } - if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { - print STDOUT "GONE: $summary\n"; - next; - } - - # single copy @ ATRC - # make a backup @ ITC - if (($Nmain == 0) && ($Nback == 1)) { - print STDOUT "SBCK: $summary\n"; - next; - } - - # multiple copies @ ATRC, but none at ITC - # make a backup at ITC, cull ATRC extras - if (($Nmain == 0) && ($Nback > 1)) { - print STDOUT "XBCK: $summary\n"; - next; - } - - # unexpected state: - print STDOUT "OOPS: $summary\n"; - next; -} - -my @hostnames = keys %hostcount; -foreach my $myname (@hostnames) { - print "host $myname = $hostcount{$myname}\n"; -} - -die "done"; - -#my $query = $dbh->prepare( $sql_get_volume_id ); - -#$query->execute( $tgt_volume ); -#my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; -#$query->finish; - - -### XX -### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; - -# load the list of files from movelist - -##XX open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; -##XX my @movefiles = ; -##XX close (MOVELIST); -##XX -##XX my $otime = $time; -##XX $time = time(); -##XX my $dtime = $time - $otime; -##XX -##XX for (my $i = 0; $i < @movefiles; $i++) { -##XX -##XX my $movefile = $movefiles[$i]; -##XX chomp $movefile; -##XX -##XX # the filename should have the following form: -##XX # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: -##XX # NN : 2-hexdigit name of the subsubdir -##XX # DDDDD : instance ID (long) -##XX # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name -##XX -##XX # I want to check two things: -##XX # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile -##XX # 2) does the vol_id of the instance_id match our vol_id ? -##XX -##XX unless (-e "$tgt_subdir/$movefile") { -##XX print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; -##XX next; -##XX } -##XX -##XX # print "movefile: $movefile\n"; -##XX -##XX my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; -##XX my $summary = "$tgt_subdir/$movefile"; -##XX -##XX if ( ! defined ($ins_id)) { -##XX print "ERROR : INVALID ID: $summary\n"; -##XX next; -##XX } -##XX -##XX if ( $ins_id =~ m/\D/ ) { -##XX print "ERROR : INVALID ID: $summary\n"; -##XX next; -##XX } -##XX $summary = "$summary : $ins_id"; -##XX -##XX # get so_id for this ins_id: -##XX my $so_id = 0; -##XX if (1) { -##XX my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; -##XX -##XX my $rowref = $dbh->selectall_arrayref( $so_id_query ); -##XX my @array = @$rowref; -##XX my $Nrows = @array; -##XX -##XX if ($Nrows == 0) { # We should find this instance -##XX print "ERROR : NO DB ENTRY: $summary\n"; -##XX next; -##XX } -##XX -##XX if ($Nrows > 1) { # We should only find one instance -##XX print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; -##XX next; -##XX } -##XX -##XX my $valref = $array[0]; -##XX my @values = @$valref; -##XX -##XX my $Nvalue = @values; -##XX -##XX if ($Nvalue != 1) { -##XX print "ERROR : INVALID DB RESPONSE: $summary\n"; -##XX next; -##XX } -##XX $so_id = $values[0]; -##XX } -##XX -##XX $summary = "$summary : $so_id"; -##XX -##XX # get user_copies for this storage_object -##XX my $Ncopies = 0; -##XX my $ext_id = ""; -##XX if (1) { -##XX my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; -##XX -##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); -##XX my @array = @$rowref; -##XX my $Nrows = @array; -##XX -##XX if ($Nrows == 0) { # user.copies is not set, skip -##XX print "SKIP: NO USER.COPIES ENTRY: $summary\n"; -##XX next; -##XX } -##XX -##XX if ($Nrows > 1) { # We should only find one instance -##XX print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; -##XX next; -##XX } -##XX -##XX my $valref = $array[0]; -##XX my @values = @$valref; -##XX -##XX my $Nvalue = @values; -##XX -##XX if ($Nvalue != 2) { -##XX print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; -##XX next; -##XX } -##XX $ext_id = $values[0]; -##XX $Ncopies = $values[1]; -##XX } -##XX -##XX $summary = "$summary : $ext_id"; -##XX -##XX if ($Ncopies < 2) { -##XX print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; -##XX next; -##XX } -##XX -##XX # check for backup instances -##XX if (1) { -##XX my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; -##XX -##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); -##XX my @array = @$rowref; -##XX my $Nrows = @array; -##XX -##XX if ($Nrows == 0) { # We should find this instance -##XX print "REPL: NO BACKUP COPY: $summary\n"; -##XX next; -##XX } -##XX -##XX ## my $valref = $array[0]; -##XX ## my @values = @$valref; -##XX ## -##XX ## my $Nvalue = @values; -##XX ## -##XX ## if ($Nvalue != 1) { -##XX ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; -##XX ## next; -##XX ## } -##XX } -##XX -##XX } # END loop over movefiles - -# example of using the return value of arrayref: - -# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); -# my @array = @$rowref; -# my $Nrows = @array; - -# print "NROWS: $Nrows\n"; - -# foreach my $valref (@array) { -# my @values = @$valref; -# foreach my $value (@values) { -# print "VAL: $value, "; -# } -# print "\n"; -# } - -sub vsystem { - # print STDERR "@_\n"; - return 0; - #my $status = system ("@_"); - #return $status; -} Index: trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (revision 42443) +++ trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (revision 42444) @@ -20,4 +20,6 @@ my $Nfail = 0; +my %failchips; +my %goodchips; foreach my $instfile (@instlist) { @@ -80,14 +82,32 @@ if ($md5tru ne $md5obs) { - print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; - $Nfail ++; + &save_failure ($inst); + if ($DEBUG > 2) { print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; } } else { - if ($DEBUG > 2) { - print STDOUT "$inst $state $md5tru = $md5obs\n"; - } + &save_goodchip ($inst); + if ($DEBUG > 2) { print STDOUT "$inst $state $md5tru = $md5obs\n"; } } } } +# save both good and fail chip lists +open (OUTFILE, ">$topdir/$dateword/goodchips.$stage.txt"); +my @goodchipURIs = keys %goodchips; +foreach my $uri (@goodchipURIs) { + print OUTFILE "$uri\n"; +} +close (OUTFILE); + +open (OUTFILE, ">$topdir/$dateword/failchips.$stage.txt"); +my @failchipURIs = keys %failchips; +foreach my $uri (@failchipURIs) { + print OUTFILE "$uri\n"; +} +close (OUTFILE); + + +## we only report an error if we could not find the md5sum files or if the inst and md5s +## files are inconsistent. In that case, we cannot check the md5s for all instances +## if the md5sum is wrong for a chip, mark that chip and save it. It will be skipped in fix_chip_locations if ($Nfail) { print STDOUT "ERROR: $Nfail errors\n"; @@ -105,5 +125,9 @@ pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; - pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -126,2 +150,42 @@ return $status; } + +sub neb_reverse { + + my (@neb_path) = split /\//, $_[0]; + my $neb_key = $neb_path[-1]; + + # Remove ins_id + $neb_key =~ s/^\d+\.//; + + # Convert colons to slashes. + $neb_key =~ s%:%/%g; + + # strip ending .fz added for compressed .fits images + $neb_key =~ s%.fz$%%; + return $neb_key; +} + +sub save_failure { + my $nebfile = &neb_reverse($_[0]); + if (defined $failchips{$nebfile}) { + $failchips{$nebfile} ++; + } else { + $failchips{$nebfile} = 1; + } + return; +} + +sub save_goodchip { + my $nebfile = &neb_reverse($_[0]); + if (defined $goodchips{$nebfile}) { + $goodchips{$nebfile} ++; + } else { + $goodchips{$nebfile} = 1; + } + return; +} + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} Index: trunk/tools/eam/rawfix.20230221/src/check_night_status.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/check_night_status.pl (revision 42443) +++ trunk/tools/eam/rawfix.20230221/src/check_night_status.pl (revision 42444) @@ -11,5 +11,5 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: check_night_status.pl --dateobs (night) --chiplist (file) +# USAGE: check_night_status.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) # we need to select targets for ITC copies of files currently only at ATRC: @@ -17,5 +17,5 @@ # for each chip, target a single host? -my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; +my ($fulldate, $dateword, $chiplist, $stage) = &parse_cmdopts; # read the list of chip_ids: @@ -36,5 +36,5 @@ foreach my $chip_id (@chip_ids) { - my $urifile = "$dateword/$chip_id.uris.txt"; + my $urifile = "$dateword/$chip_id.uris.$stage.txt"; open (FILE, "$urifile"); my @lines = ; @@ -110,9 +110,15 @@ sub parse_cmdopts { - my ($dateobs, $chiplist); - GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + my ($dateobs, $chiplist, $stage); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'stage=s' => \$stage) || pod2usage(2); pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -127,5 +133,10 @@ my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - return ($fulldate, $dateword, $chiplist); + return ($fulldate, $dateword, $chiplist, $stage); } + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42443) +++ trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42444) @@ -12,5 +12,5 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) +# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) # we need to select targets for ITC copies of files currently only at ATRC: @@ -25,10 +25,11 @@ # choose ITC machines on which to place backup copies: -my @volarray = &get_neb_volumes; +my @vol_copy = &get_neb_volumes_copy; # array of neb hosts to copy data to at ITC +my %vol_good = &get_neb_volumes_good; # hash of valid neb hosts my $volnum = 0; my $Ngood = 0; +my $Ndead = 0; my $Ngone = 0; -my $Ndead = 0; my $Nsing = 0; my $Ndups = 0; @@ -40,10 +41,39 @@ my $Ncul3 = 0; +# load the lists of good and fail chips +my $prev = $stage - 1; +my $goodfile = "$topdir/$dateword/goodchips.$prev.txt"; +my %goodchips; +if (-e $goodfile) { + open (GOODFILE, $goodfile); + my @lines = ; + close (GOODFILE); + + foreach my $line (@lines) { + chomp $line; + $goodchips{$line} = 1; + } +} +my $failfile = "$topdir/$dateword/failchips.$prev.txt"; +my %failchips; +if (-e $failfile) { + open (FAILFILE, $failfile); + my @lines = ; + close (FAILFILE); + + foreach my $line (@lines) { + chomp $line; + $failchips{$line} = 1; + } +} + foreach my $chip_id (@chip_ids) { - my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; - open (FILE, $urifile); - my @lines = ; - close (FILE); + # read from chip_id file, update to next stage number + my $urifile = "$topdir/$dateword/$chip_id.uris.$stage.txt"; + if (! -e $urifile) { die "missing file $urifile\n"; } + open (URIFILE, $urifile); + my @lines = ; + close (URIFILE); foreach my $line (@lines) { @@ -61,4 +91,11 @@ my $md5sum = $words[3]; + my $haveMD5 = 0; + my $goodMD5 = 0; + if ($goodchips{$ext_id}) { $haveMD5 = 1; $goodMD5 = 1; } + if ($failchips{$ext_id}) { $haveMD5 = 1; } + + + # nothing to be done, or need be done for these: if ($state eq "GOOD:") { $Ngood++; next; } if ($state eq "DEAD:") { $Ndead++; next; } @@ -66,4 +103,5 @@ # one copy at ITC: neb-replicate will create a copy at ATRC + # does not depend on existence of MD5sum if ($state eq "SING:") { my $status = &vsystem ("neb-replicate $ext_id"); @@ -79,6 +117,7 @@ # one copy at ATRC: neb-replicate will create a copy at ITC iff we supply the target volume + # does not depend on existence of MD5sum if ($state eq "SBCK:") { - my $tgtvol = $volarray[$volnum]; + my $tgtvol = $vol_copy[$volnum]; my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); if ($status) { print STDERR "FAIL SBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } @@ -92,12 +131,68 @@ } - # these are done in stage 2: - if ($state eq "DUPS:") { $Ndups++; next; } - if ($state eq "XBCK:") { $Nxbck++; next; } - - # these are done in stage 3: - if ($state eq "CUL1:") { $Ncul1++; next; } - if ($state eq "CUL2:") { $Ncul2++; next; } - if ($state eq "CUL3:") { $Ncul3++; next; } + # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies + # skip if we do not know the state of the md5sum + if ($state eq "DUPS:") { + if ($haveMD5 && $goodMD5) { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + } + $Ndups++; + next; + } + + # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume + # skip if we do not know the state of the md5sum + if ($state eq "XBCK:") { + if ($haveMD5 && $goodMD5) { + my $tgtvol = $vol_copy[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + } + $Nxbck++; + next; + } + + # one copy at ITC: neb-replicate will create a copy at ATRC + # skip if we do not know the state of the md5sum + if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:")) { + + if ($haveMD5 && $goodMD5) { + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($ext_id); + + my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id); + + if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n"; next; } + if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; } + + my $NcullTotal = @$cull_list_r; + foreach my $instance (@$cull_list_r) { + my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|; + unless (defined $vol_good{$volname}) { die "ERROR: invalid volume name $volname\n"; } + # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n"; + my $status = &vsystem ("neb-cull --volume $volname $ext_id"); + if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; } + } + # print STDERR "Ncull Total $NcullTotal\n"; + } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } } $volnum ++; @@ -136,6 +231,60 @@ } -# get the list of chip neb paths for one chip -sub get_neb_volumes { +sub get_chip_instances { + + my $ext_id = $_[0]; + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# select 1 instance with xattr = 0, 1 with xattr = 1. +# the rest should be culled. + +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $main_keep = ""; + my $back_keep = ""; + my @cull_list = (); + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + # the number of returned values per row must match the query above (in get_chip_instances) + my $Nvalue = @values; + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; } + if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; } + + # if we make it here, we already have an instance at this location + push (@cull_list, $instance); + } + return ($main_keep, $back_keep, \@cull_list); +} + +# get the list of neb hosts to copy data to at ITC +sub get_neb_volumes_copy { my @volarray; @@ -164,4 +313,32 @@ } +# get the list of valid neb hosts (ITC or ATRC) +sub get_neb_volumes_good { + + my %volarray; + + my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + $volarray{$values[0]} = 1; + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return %volarray; +} + # read the list of chip_ids: sub load_chip_list { @@ -194,7 +371,11 @@ pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; - pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -248,6 +429,11 @@ sub vsystem { print STDERR "@_\n"; - #return 0; + # return 0; my $status = system ("@_"); return $status; } + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage2.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage2.pl (revision 42443) +++ (revision ) @@ -1,253 +1,0 @@ -#! /usr/bin/env perl - -my $DEBUG = 4; -my $time = time(); - -use DBI; -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) - -# we need to select targets for ITC copies of files currently only at ATRC: -# choose a list of the neb hosts with lots of space (by percent or bytes?) -# for each chip, target a single host? - -my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; -my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); - -# read the list of chip_ids: -my @chip_ids = &load_chip_list ($chiplist); - -# choose ITC machines on which to place backup copies: -my @volarray = &get_neb_volumes; -my $volnum = 0; - -my $Ngood = 0; -my $Ndead = 0; -my $Ngone = 0; -my $Nsing = 0; -my $Ndups = 0; -my $Nsbck = 0; -my $Nxbck = 0; - -my $Ncul1 = 0; -my $Ncul2 = 0; -my $Ncul3 = 0; - -foreach my $chip_id (@chip_ids) { - - my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; - open (FILE, $urifile); - my @lines = ; - close (FILE); - - foreach my $line (@lines) { - chomp $line; - if ($line =~ /^\s*#/) { next; } # skip lines with leading # - my @words = split (/\s+/,$line); - - # we expect lines of the form: STATE PATH - # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum - if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } - - my $state = $words[0]; - my $ext_id = $words[1]; - my $datast = $words[2]; - my $md5sum = $words[3]; - - if ($state eq "GOOD:") { $Ngood++; next; } - if ($state eq "DEAD:") { $Ndead++; next; } - if ($state eq "GONE:") { $Ngone++; next; } - - # these were done in stage 1: - if ($state eq "SING:") { $Nsing++; next; } # one copy at ITC: should not happen at stage 2 - if ($state eq "SBCK:") { $Nsbck++; next; } # one copy at ATRC: should not happen at stage 2 - - # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies - if ($state eq "DUPS:") { - my $status = &vsystem ("neb-replicate $ext_id"); - if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; } - my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); - unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } - open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); - print OUTFILE "$newfile $datast $md5sum\n"; - close (OUTFILE); - $Ndups++; - next; - } - - # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume - if ($state eq "XBCK:") { - my $tgtvol = $volarray[$volnum]; - my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); - if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } - my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); - unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } - open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); - print OUTFILE "$newfile $datast $md5sum\n"; - close (OUTFILE); - $Nxbck++; - next; - } - - # these are done in stage 3: - if ($state eq "CUL1:") { $Ncul1++; next; } - if ($state eq "CUL2:") { $Ncul2++; next; } - if ($state eq "CUL3:") { $Ncul3++; next; } - } - $volnum ++; -} - -print "Ngood: $Ngood\n"; -print "Ngone: $Ngone\n"; -print "Ndead: $Ndead\n"; -print "Nsing: $Nsing\n"; -print "Ndups: $Ndups\n"; -print "Nsbck: $Nsbck\n"; -print "Nxbck: $Nxbck\n"; -print "Ncul1: $Ncul1\n"; -print "Ncul2: $Ncul2\n"; -print "Ncul3: $Ncul3\n"; - -exit 0; - -sub get_new_instance { - - my $ext_id = $_[0]; - - ## the first instance should be the last one created: - my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; - my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); - my @tmparray = @$rowref; - - my $valref = $tmparray[0]; - my @values = @$valref; - - my $uri = $values[0]; - my $host = $values[1]; - my $xattr = $values[2]; - - return ($uri, $host, $xattr); -} - -# get the list of chip neb paths for one chip -sub get_neb_volumes { - - my @volarray; - - my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; - if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } - - my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); - my @tmparray = @$rowref; - my $Nrows = @tmparray; - - if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } - - foreach my $valref (@tmparray) { - my @values = @$valref; - # there should only be a single value (only name selected above) - my $Nvalues = @values; - if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } - if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } - - push (@volarray, $values[0]); - if ($DEBUG > 4) { print "$values[0]\n"; } - } - - return @volarray; -} - -# read the list of chip_ids: -sub load_chip_list { - my $chiplist = $_[0]; - - my @chip_ids; - - open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; - - my @lines = ; - - foreach my $line (@lines) { - chomp $line; - if ($line =~ /^\s*#/) { next; } # skip lines with leading # - my @words = split (/\s+/,$line); - - if (@words != 1) { die "error in chip list $line\n"; } - push (@chip_ids, $words[0]); - } - close (CHIPS); - - return @chip_ids; -} - -sub parse_cmdopts { - - my ($dateobs, $chiplist, $stage, $topdir); - GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); - - pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; - pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; - pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; - pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; - - # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) - - my ($date_year, $date_month, $date_day) = split "/",$dateobs; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); - - my $fulldate = "$date_year/$date_month/$date_day"; - my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - - return ($fulldate, $dateword, $chiplist, $stage, $topdir); -} - -# parse the nebulous.config file: -sub parse_db_config { - - my $conffile = $_[0]; - - my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); - - if (-e $conffile) { - open(IN,$conffile); - while() { - chomp; - if ($_ =~ /^\s*#/) { next; } # skip lines with leading # - if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } - } - close(IN); - print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; - } - - pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; - pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; - pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; - pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; - pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; - - my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; - if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } - my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - - return ($neb_dbh); -} - -sub vsystem { - print STDERR "@_\n"; - # return 0; - my $status = system ("@_"); - return $status; -} Index: trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage3.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage3.pl (revision 42443) +++ (revision ) @@ -1,286 +1,0 @@ -#! /usr/bin/env perl - -my $DEBUG = 4; -my $time = time(); - -use DBI; -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -# USAGE: fix_chip_locations_stage3.pl --dateobs (night) --chiplist (file) - -my ($fulldate, $dateword, $chiplist, $topdir) = &parse_cmdopts; -my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); - -# read the list of chip_ids: -my @chip_ids = &load_chip_list ($chiplist); -my %volarray = &get_neb_volumes; - -# choose ITC machines on which to place backup copies: -my $Ngood = 0; -my $Ngone = 0; -my $Ndead = 0; -my $Nsing = 0; -my $Ndups = 0; -my $Nsbck = 0; -my $Nxbck = 0; -my $Ncul1 = 0; -my $Ncul2 = 0; -my $Ncul3 = 0; - -foreach my $chip_id (@chip_ids) { - - my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; - open (FILE, $urifile); - my @lines = ; - close (FILE); - - foreach my $line (@lines) { - chomp $line; - if ($line =~ /^\s*#/) { next; } # skip lines with leading # - my @words = split (/\s+/,$line); - - # we expect lines of the form: STATE PATH - # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum - if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } - - my $state = $words[0]; - my $ext_id = $words[1]; - my $datast = $words[2]; - my $md5sum = $words[3]; - - if ($state eq "GOOD:") { $Ngood++; next; } - if ($state eq "DEAD:") { $Ndead++; next; } - if ($state eq "GONE:") { $Ngone++; next; } - - # these are done in stage 1: - if ($state eq "SING:") { $Nsing++; next; } - if ($state eq "SBCK:") { $Nsbck++; next; } - - # these were replicated in stage 2, so actually match one of the CUL1 or CUL2 entries - # if ($state eq "DUPS:") { $Ndups++; next; } - # if ($state eq "XBCK:") { $Nxbck++; next; } - - # these are done in stage 3: - # if ($state eq "CUL1:") { $Ncul1++; next; } - # if ($state eq "CUL2:") { $Ncul2++; next; } - # if ($state eq "CUL3:") { $Ncul3++; next; } - - # one copy at ITC: neb-replicate will create a copy at ATRC - if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:") || ($state eq "DUPS:") || ($state eq "XBCK:")) { - - # get info for all instances of this chip - # returns a reference to the array of instance data - # also strips neb://hostname.volnum/ from nebname and returns as ext_id - my ($inforows, $ext_id) = &get_chip_instances ($ext_id); - - my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id); - - if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n"; next; } - if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; } - - my $NcullTotal = @$cull_list_r; - foreach my $instance (@$cull_list_r) { - my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|; - unless (defined $volarray{$volname}) { die "ERROR: invalid volume name $volname\n"; } - # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n"; - my $status = &vsystem ("neb-cull --volume $volname $ext_id"); - if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; } - } - # print STDERR "Ncull Total $NcullTotal\n"; - - # these are done in stage 3: - if ($state eq "CUL1:") { $Ncul1++; next; } - if ($state eq "CUL2:") { $Ncul2++; next; } - if ($state eq "CUL3:") { $Ncul3++; next; } - } - } -} - -print "Ngood: $Ngood\n"; -print "Ngone: $Ngone\n"; -print "Ndead: $Ndead\n"; -print "Nsing: $Nsing\n"; -print "Ndups: $Ndups\n"; -print "Nsbck: $Nsbck\n"; -print "Nxbck: $Nxbck\n"; -print "Ncul1: $Ncul1\n"; -print "Ncul2: $Ncul2\n"; -print "Ncul3: $Ncul3\n"; - -exit 0; - -sub get_chip_instances { - - my $ext_id = $_[0]; - - my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; - my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); - - return ($rowref, $ext_id); -} - -# select 1 instance with xattr = 0, 1 with xattr = 1. -# the rest should be culled. - -sub check_instance_xattr { - - my $rowref = $_[0]; - my $ext_id = $_[1]; - - my @inforows = @$rowref; - my $Nrows = @inforows; - - if ($DEBUG > 5) { print "found $Nrows instances\n"; } - - # get user_copies for this storage_object - my $main_keep = ""; - my $back_keep = ""; - my @cull_list = (); - - # xattr = 0 means instance @ ITC (main) - # xattr = 1 means instance @ ATRC (back) - for (my $j = 0; $j < $Nrows; $j ++) { - my $valref = $inforows[$j]; - my @values = @$valref; - - # the number of returned values per row must match the query above (in get_chip_instances) - my $Nvalue = @values; - if ($Nvalue != 5) { - print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; - next; - } - - my $instance = $values[0]; - my $hostname = $values[1]; - my $xattr = $values[3]; - - if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; } - if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; } - - # if we make it here, we already have an instance at this location - push (@cull_list, $instance); - } - return ($main_keep, $back_keep, \@cull_list); -} - -# read the list of chip_ids: -sub load_chip_list { - my $chiplist = $_[0]; - - my @chip_ids; - - open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; - - my @lines = ; - - foreach my $line (@lines) { - chomp $line; - if ($line =~ /^\s*#/) { next; } # skip lines with leading # - my @words = split (/\s+/,$line); - - if (@words != 1) { die "error in chip list $line\n"; } - push (@chip_ids, $words[0]); - } - close (CHIPS); - - return @chip_ids; -} - -# -sub parse_cmdopts { - - my ($dateobs, $chiplist, $topdir); - GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir) || pod2usage(2); - - pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; - pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; - pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; - - # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) - - my ($date_year, $date_month, $date_day) = split "/",$dateobs; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); - - my $fulldate = "$date_year/$date_month/$date_day"; - my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - - return ($fulldate, $dateword, $chiplist, $topdir); -} - -# get the list of chip neb paths for one chip -sub get_neb_volumes { - - my %volarray; - - my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))"; - if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } - - my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); - my @tmparray = @$rowref; - my $Nrows = @tmparray; - - if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } - - foreach my $valref (@tmparray) { - my @values = @$valref; - # there should only be a single value (only name selected above) - my $Nvalues = @values; - if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } - if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } - - $volarray{$values[0]} = 1; - if ($DEBUG > 4) { print "$values[0]\n"; } - } - - return %volarray; -} - -# parse the nebulous.config file: -sub parse_db_config { - - my $conffile = $_[0]; - - my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); - - if (-e $conffile) { - open(IN,$conffile); - while() { - chomp; - if ($_ =~ /^\s*#/) { next; } # skip lines with leading # - if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } - } - close(IN); - print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; - } - - pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; - pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; - pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; - pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; - pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; - - my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; - if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } - my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - - return ($neb_dbh); -} - -sub vsystem { - print STDERR "@_\n"; - # return 0; - my $status = system ("@_"); - return $status; -} Index: trunk/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl (revision 42443) +++ (revision ) @@ -1,50 +1,0 @@ -#! /usr/bin/env perl - -die "XXX not done yet"; - -my $DEBUG = 4; -my $time = time(); - -use DBI; -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -# USAGE: get_md5s_dateobs.pl --dateobs (night) - -my ($fulldate, $dateword) = &parse_cmdopts; - -# first, ensure the dateobs directory is - - - -sub parse_cmdopts { - - my ($dateobs); - GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2); - - pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; - - # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) - - my ($date_year, $date_month, $date_day) = split "/",$dateobs; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); - - my $fulldate = "$date_year/$date_month/$date_day"; - my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - - return ($fulldate, $dateword); -} - -sub vsystem { - print STDERR "@_\n"; - #return 0; - my $status = system ("@_"); - return $status; -} Index: trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl (revision 42443) +++ trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl (revision 42444) @@ -11,5 +11,5 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc) +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (0,1,etc) my ($myhost, $input, $stage) = &parse_cmdopts; @@ -26,11 +26,12 @@ if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } + +# read in both instance +my $out_hash_r; if (-e $output) { - # check that all instances are in the output file? - print "$output exists, skipping\n"; - exit 0; + $out_hash_r = &load_output_list ($output); } -open (OUTFILE, ">$output"); +open (OUTFILE, ">>$output"); # foreach my $instance (@$ins_list_r) { @@ -41,7 +42,16 @@ my $md5sum_tru = $md5_list_r->[$i]; + # if the instance is already in the output file, skip it + if (defined $out_hash_r) { + my $name = $out_hash_r->{$instance}; + if (defined $name) { + print STDERR "skipping $instance (already in output)\n"; + next; + } + } + my $filename = $instance; $filename =~ s|file://||; - + # check for the existence of the file first: unless (-e $filename) { @@ -63,5 +73,5 @@ exit 0; -# read the list of chip_ids: +# read the instance info sub load_instance_list { my $infile = $_[0]; @@ -88,4 +98,27 @@ } +# read the instance list in the output file +sub load_output_list { + my $infile = $_[0]; + + my (%out_hash); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 4) { die "error in input $infile : $line\n"; } + $out_hash{$words[0]} = 1; + } + close (FILE); + + return (\%out_hash); +} + sub parse_cmdopts { @@ -95,5 +128,9 @@ pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; - pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } return ($myhost, $input, $stage); @@ -106,2 +143,7 @@ return $status; } + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: trunk/tools/eam/rawfix.20230221/src/get_md5s_instances_stage2.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/get_md5s_instances_stage2.pl (revision 42443) +++ (revision ) @@ -1,140 +1,0 @@ -#!/usr/bin/env perl - -my $DEBUG = 4; -my $time = time(); - -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc) - -my ($myhost, $input, $stage) = &parse_cmdopts; - -# check that we are running this script on 'myhost' -my $truhost = `hostname -s`; chomp $truhost; -if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } - -# read the list of instances: -my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); - -my $output = $input; -$output =~ s/inst.$stage/md5s/; - -if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } - -# read in both instance -my $out_hash_r; -if (-e $output) { - $out_hash_r = &load_output_list ($output); -} - -open (OUTFILE, ">>$output"); - -# foreach my $instance (@$ins_list_r) { -for (my $i = 0; $i < @$ins_list_r; $i ++) { - - my $instance = $ins_list_r->[$i]; - my $data_state = $dst_list_r->[$i]; - my $md5sum_tru = $md5_list_r->[$i]; - - # if the instance is already in the output file, skip it - if (defined $out_hash_r) { - my $name = $out_hash_r->{$instance}; - if (defined $name) { - print STDERR "skipping $instance (already in output)\n"; - next; - } - } - - my $filename = $instance; - $filename =~ s|file://||; - - # check for the existence of the file first: - unless (-e $filename) { - print OUTFILE "$instance $data_state $md5sum_tru XXX\n"; - next; - } - - my $line; - if (0) { - $line = "fakevalue $filename"; # XXX TEST: disable calculation: - } else { - $line = `md5sum $filename`; - } - - my ($md5sum, $rawfile) = split (" ", $line); - print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n"; -} -close OUTFILE; -exit 0; - -# read the instance info -sub load_instance_list { - my $infile = $_[0]; - - my (@ins_list, @dst_list, @md5_list); - - open(FILE,$infile) or die "cannot open file $infile\n"; - - my @lines = ; - - foreach my $line (@lines) { - chomp $line; - if ($line =~ /^\s*#/) { next; } # skip lines with leading # - my @words = split (/\s+/,$line); - - if (@words != 3) { die "error in input $infile : $line\n"; } - push (@ins_list, $words[0]); - push (@dst_list, $words[1]); - push (@md5_list, $words[2]); - } - close (FILE); - - return (\@ins_list, \@dst_list, \@md5_list); -} - -# read the instance list in the output file -sub load_output_list { - my $infile = $_[0]; - - my (%out_hash); - - open(FILE,$infile) or die "cannot open file $infile\n"; - - my @lines = ; - - foreach my $line (@lines) { - chomp $line; - if ($line =~ /^\s*#/) { next; } # skip lines with leading # - my @words = split (/\s+/,$line); - - if (@words != 4) { die "error in input $infile : $line\n"; } - $out_hash{$words[0]} = 1; - } - close (FILE); - - return (\%out_hash); -} - -sub parse_cmdopts { - - my ($myhost, $input, $stage); - GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2); - - pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; - pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; - pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; - - return ($myhost, $input, $stage); -} - -sub vsystem { - print STDERR "@_\n"; - #return 0; - my $status = system ("@_"); - return $status; -} Index: trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip_s3.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.ckchip_s3.pt (revision 42443) +++ (revision ) @@ -1,117 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the initial chip instance checks -## use the top-level rawfix.pt to load - -# these tasks run the steps: -# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt -# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) - -if (not($?RAWFIX_CKCHIP_S3_INIT)) - book init ckchip_s3.book - $RAWFIX_CKCHIP_S3_INIT = 1 -end - -# select nights in state ckchip_s3.new -# output: ckchip_s3.book -task nights.ckchip_s3.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/ckchip_s3.load.stdout.log - stderr $LOGDIR/ckchip_s3.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - command check.rawfix.sh state ckchip_s3.new - end - - # need to delete DONE entries here so the book is not repopulated - # by already-running load tasks - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip_s3.book - book delpage ckchip_s3.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task nights.ckchip_s3.run - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - host anyhost - npending 1 - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages ckchip_s3.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in ckchip_s3.book (pantaskState == INIT) - book getpage ckchip_s3.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - stdout $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 - stderr $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 - - book setword ckchip_s3.book $pageName pantaskState RUN - book getword ckchip_s3.book $pageName dateobs -var DATEOBS - option $pageName - - substr $DATEOBS 0 4 YEAR - substr $DATEOBS 4 2 MONTH - substr $DATEOBS 6 2 DAY - - sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - - command $mypath/check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v1 - end - - # SUCCESS - task.exit 0 - book setword ckchip_s3.book $options:0 pantaskState DONE - exec update.rawfix.sh $options:0 ckchip_s3.done - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword ckchip_s3.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 ckchip_s3.fail - end - - task.exit crash - showcommand crash - book setword ckchip_s3.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 ckchip_s3.fail - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword ckchip_s3.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 ckchip_s3.fail - end -end - Index: trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt (revision 42443) +++ (revision ) @@ -1,119 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the initial chip instance checks -## use the top-level rawfix.pt to load - -# these tasks run the steps: -# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt -# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) - -if (not($?RAWFIX_FIXCHIP_S2_INIT)) - book init fixchip_s2.book - $RAWFIX_FIXCHIP_S2_INIT = 1 -end - -# select nights in state fixchip_s2.new -# output: fixchip_s2.book -task nights.fixchip_s2.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/fixchip_s2.load.stdout.log - stderr $LOGDIR/fixchip_s2.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - # command check.rawfix.sh state ckchip.done - command check.rawfix.sh state fixchip_s2.new - end - - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s2.book - book delpage fixchip_s2.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task nights.fixchip_s2.run - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 1800 - host anyhost - npending 1 - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages fixchip_s2.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in fixchip_s2.book (pantaskState == INIT) - book getpage fixchip_s2.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword fixchip_s2.book $pageName pantaskState RUN - book getword fixchip_s2.book $pageName dateobs -var DATEOBS - option $pageName - - substr $DATEOBS 0 4 YEAR - substr $DATEOBS 4 2 MONTH - substr $DATEOBS 6 2 DAY - sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - - # does the output go here? - stdout $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 - stderr $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 - - command $mypath/fix_chip_locations_stage2.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 - end - - ## XX delete the page from the book if done? - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - # use the queuesize (Nhosts) to track how many jobs are done / to be done - book setword fixchip_s2.book $options:0 pantaskState DONE - exec update.rawfix.sh $options:0 fixchip_s2.done - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword fixchip_s2.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 fixchip_s2.fail - end - - task.exit crash - showcommand crash - book setword fixchip_s2.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 fixchip_s2.fail - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword fixchip_s2.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 fixchip_s2.fail - end -end - Index: trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s3.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s3.pt (revision 42443) +++ (revision ) @@ -1,119 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the initial chip instance checks -## use the top-level rawfix.pt to load - -# these tasks run the steps: -# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt -# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) - -if (not($?RAWFIX_FIXCHIP_S3_INIT)) - book init fixchip_s3.book - $RAWFIX_FIXCHIP_S3_INIT = 1 -end - -# select nights in state fixchip_s3.new -# output: fixchip_s3.book -task nights.fixchip_s3.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/fixchip_s3.load.stdout.log - stderr $LOGDIR/fixchip_s3.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - # command check.rawfix.sh state ckchip.done - command check.rawfix.sh state fixchip_s3.new - end - - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s3.book - book delpage fixchip_s3.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task nights.fixchip_s3.run - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 1800 - host anyhost - npending 1 - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages fixchip_s3.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in fixchip_s3.book (pantaskState == INIT) - book getpage fixchip_s3.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword fixchip_s3.book $pageName pantaskState RUN - book getword fixchip_s3.book $pageName dateobs -var DATEOBS - option $pageName - - substr $DATEOBS 0 4 YEAR - substr $DATEOBS 4 2 MONTH - substr $DATEOBS 6 2 DAY - sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - - # does the output go here? - stdout $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 - stderr $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 - - command $mypath/fix_chip_locations_stage3.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt - end - - ## XX delete the page from the book if done? - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - # use the queuesize (Nhosts) to track how many jobs are done / to be done - book setword fixchip_s3.book $options:0 pantaskState DONE - exec update.rawfix.sh $options:0 fixchip_s3.done - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword fixchip_s3.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 fixchip_s3.fail - end - - task.exit crash - showcommand crash - book setword fixchip_s3.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 fixchip_s3.fail - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword fixchip_s3.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 fixchip_s3.fail - end -end - Index: trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk_s2.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk_s2.pt (revision 42443) +++ (revision ) @@ -1,118 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the initial chip instance checks -## use the top-level rawfix.pt to load - -# these tasks run the steps: -# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD - -if (not($?RAWFIX_MD5CHK_S2_INIT)) - book init md5chk_s2.book - $RAWFIX_MD5CHK_S2_INIT = 1 -end - -# select nights in state md5chk_s2.new -# output: md5chk_s2.book -task nights.md5chk_s2.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/md5chk_s2.load.stdout.log - stderr $LOGDIR/md5chk_s2.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - # command check.rawfix.sh state ckchip.done - command check.rawfix.sh state md5chk_s2.new - end - - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk_s2.book - book delpage md5chk_s2.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task nights.md5chk_s2.run - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - host anyhost - npending 1 - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages md5chk_s2.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in md5chk_s2.book (pantaskState == INIT) - book getpage md5chk_s2.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword md5chk_s2.book $pageName pantaskState RUN - book getword md5chk_s2.book $pageName dateobs -var DATEOBS - option $pageName - - substr $DATEOBS 0 4 YEAR - substr $DATEOBS 4 2 MONTH - substr $DATEOBS 6 2 DAY - sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - - # does the output go here? - stdout $mypath/$DATEOBS/log.md5chk_s2.v0 - stderr $mypath/$DATEOBS/log.md5chk_s2.v0 - - command $mypath/check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage v0 - end - - ## XX delete the page from the book if done? - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - # use the queuesize (Nhosts) to track how many jobs are done / to be done - book setword md5chk_s2.book $options:0 pantaskState DONE - exec update.rawfix.sh $options:0 md5chk_s2.done - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword md5chk_s2.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 md5chk_s2.fail - end - - task.exit crash - showcommand crash - book setword md5chk_s2.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 md5chk_s2.fail - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword md5chk_s2.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 md5chk_s2.fail - end -end - Index: trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt (revision 42443) +++ (revision ) @@ -1,283 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the md5sum_s2 calculations -## use the top-level rawfix.pt to load - -#### the scripts below take the input files (YYYYMMDD/*.inst.txt) -#### and run get_md5s_instances.pl on the hosts associated with those lists -#### the outputs are files (YYYYMMDD/*.md5s.txt) - -if (not($?RAWFIX_MD5SUM_S2_INIT)) - book init md5sum_s2.book - book init md5host_s2.book - $RAWFIX_MD5SUM_S2_INIT = 1 -end - -# select nights which are ready for md5sum_s2 calcs -# output: md5sum_s2.book -task nights.md5sum_s2.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/md5sum_s2.load.stdout.log - stderr $LOGDIR/md5sum_s2.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - command check.rawfix.sh state md5sum_s2.new - end - - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum_s2.book - book delpage md5sum_s2.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -# output: md5host_s2.book -task nights.md5sum_s2.loadhosts - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - host local - npending 1 - - stdout $LOGDIR/md5sum_s2.loadhosts.stdout.log - stderr $LOGDIR/md5sum_s2.loadhosts.stderr.log - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages md5sum_s2.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in md5sum_s2.book (pantaskState == INIT) - book getpage md5sum_s2.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword md5sum_s2.book $pageName pantaskState RUN - book getword md5sum_s2.book $pageName dateobs -var DATEOBS - option $pageName - # save the page so we can use it below - - # write an entry point in the log for this night - sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $DATEOBS - date -var mydate - exec echo "# start runhosts $mydate" >> $tmpline - - command $mypath/get_hosts_md5s.sh $mypath $DATEOBS v0 - end - - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queuesize stdout -var Nhosts - queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host_s2.book - book setword md5sum_s2.book $options:0 NHOST {$Nhosts - 1} - book setword md5sum_s2.book $options:0 NDONE 0 - # use the queuesize (Nhosts) to track how many jobs are done / to be done - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword md5sum_s2.book $options:0 pantasksQuality FAIL - end - - task.exit crash - showcommand crash - book setword md5sum_s2.book $options:0 pantasksQuality CRASH - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT - end -end - -# output: md5host_s2.book -task nights.md5sum_s2.runhosts - - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages md5host_s2.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new entries in md5host_s2.book (pantaskState == INIT) - book getpage md5host_s2.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - book setword md5host_s2.book $pageName pantaskState RUN - book getword md5host_s2.book $pageName dateobs -var DATEOBS - book getword md5host_s2.book $pageName md5host -var MD5HOST - - option $pageName $DATEOBS - # save DATEOBS so we can find the md5sum_s2.book entry - - # does the output go here? - stdout $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 - stderr $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 - - # XXX TEST: $USE_HOST = ipp060 - - $USE_HOST = $MD5HOST - host -required $USE_HOST - ## check if this host needs to be launched - if (not($?host:$USE_HOST)) - echo "host not defined, creating : $USE_HOST" - $host:$USE_HOST = 0 - end - if (not($host:$USE_HOST)) - control host add $USE_HOST - $host:$USE_HOST = 1 - end - - ## even when testing, use the original hostname here: - $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt - - # need to make sure this will work on the ipp b-nodes: - # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs - periods -exec 0.05 - command $mypath/get_md5s_instances_stage2.pl --hostname $USE_HOST --input $input --stage v0 - end - - # dump stdout / stderr to a file on failure? - task.exit 0 - nights.md5sum_s2.exit OK - end - - # exit values other than 0: - task.exit default - nights.md5sum_s2.exit FAIL - showcommand failure - end - - task.exit crash - nights.md5sum_s2.exit CRASH - showcommand crash - end - - # operation times out? - task.exit timeout - nights.md5sum_s2.exit TIMEOUT - showcommand timeout - end -end - -# output: md5host_s2.book -task nights.md5sum_s2.clean - - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - npending 1 - - stdout $LOGDIR/md5sum_s2.clean.stdout.log - stderr $LOGDIR/md5sum_s2.clean.stderr.log - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages md5sum_s2.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for completed nights in md5sum_s2.book (pantaskState == DONE) - book getpage md5sum_s2.book 0 -var pageName -key pantaskState STOP - if ("$pageName" == "NULL") break - - # book listpage md5sum_s2.book $pageName - - # book setword md5sum_s2.book $pageName pantaskState RUN - book getword md5sum_s2.book $pageName dateobs -var DATEOBS - book getword md5sum_s2.book $pageName pantasksQuality -var QUALITY - - option $pageName - # save the page so we can use it below - - if ("$QUALITY" == "OK") - command update.rawfix.sh $DATEOBS md5sum_s2.done - else - command update.rawfix.sh $DATEOBS md5sum_s2.fail - end - end - - task.exit 0 - # now that the entry is set to done in the db, remove the pantasks book entry - echo "cleaning this page: $options:0" - book setword md5sum_s2.book $options:0 pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword md5sum_s2.book $options:0 pantasksQuality FAIL - end - - task.exit crash - showcommand crash - book setword md5sum_s2.book $options:0 pantasksQuality CRASH - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT - end -end - -######################### UTILITIES ################################# - -macro nights.md5sum_s2.exit - if ($0 != 2) - echo "USAGE: nights.md5sum_s2.exit (quality)" - break - end - - book setword md5host_s2.book $options:0 pantaskState DONE - book getword md5sum_s2.book $options:1 NHOST -var Nhosts - book getword md5sum_s2.book $options:1 NDONE -var Ndone - $Ndone ++ - book setword md5sum_s2.book $options:1 NDONE $Ndone - if ("$1" != "OK") - book setword md5sum_s2.book $options:1 pantasksQuality $1 - sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $options:1 - exec echo "failure $1 $options:0" >> $tmpline - else - exec echo "success $1 $options:0" >> $tmpline - end - if ($Ndone == $Nhosts) - book setword md5sum_s2.book $options:1 pantaskState STOP - end - book delpage md5host_s2.book $options:0 -end - Index: trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s1.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s1.pt (revision 42443) +++ (revision ) @@ -1,126 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the initial chip instance checks -## use the top-level rawfix.pt to load - -# these tasks run the steps: -# -- rsync -auv $SRCDIR/ $TGTDIR/ - -$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221 -$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221 - -if (not($?RAWFIX_RSYNC_S1_INIT)) - book init rsync_s1.book - $RAWFIX_RSYNC_S1_INIT = 1 -end - -# select nights in state rsync_s1.new -# output: rsync_s1.book -task nights.rsync_s1.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/rsync_s1.load.stdout.log - stderr $LOGDIR/rsync_s1.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - # command check.rawfix.sh state ckchip.done - command check.rawfix.sh state rsync_s1.new - end - - # need to delete DONE entries here so the book is not repopulated - # by already-running load tasks - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s1.book - book delpage rsync_s1.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -###########################################################3333333 - -task nights.rsync_s1.run - - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - npending 1 - - stdout $LOGDIR/rsync_s1.run.stdout.log - stderr $LOGDIR/rsync_s1.run.stderr.log - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages rsync_s1.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in rsync_s1.book (pantaskState == INIT) - book getpage rsync_s1.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - # echo "got a valid entry: $pageName" - # book listpage rsync_s1.book $pageName - - book setword rsync_s1.book $pageName pantaskState RUN - book getword rsync_s1.book $pageName dateobs -var DATEOBS - option $pageName - - # send output to a file in the DATEOBS directory - stdout $DATEOBS/rsync_s1.log - stderr $DATEOBS/rsync_s1.log - - command rsync -auv $SRCDIR/$DATEOBS/ $TGTDIR/$DATEOBS/ - end - - ## do not delete the page from the book if done here - ## deletion has to happen after the .load - task.exit 0 - echo "--- stdout : nights.rsync_s1.run ---" - queueprint stdout - exec update.rawfix.sh $options:0 rsync_s1.done - book setword rsync_s1.book $options:0 pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword rsync_s1.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 rsync_s1.fail - end - - task.exit crash - showcommand crash - book setword rsync_s1.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 rsync_s1.fail - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword rsync_s1.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 rsync_s1.fail - end -end - Index: trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s2.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.rsync_s2.pt (revision 42443) +++ (revision ) @@ -1,124 +1,0 @@ -## pantasks scripts to manage the raw exposure data repair / cleanup -## this is for the initial chip instance checks -## use the top-level rawfix.pt to load - -# these tasks run the steps: -# -- rsync -auv $SRCDIR/ $TGTDIR/ - -$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221 -$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221 - -if (not($?RAWFIX_RSYNC_S2_INIT)) - book init rsync_s2.book - $RAWFIX_RSYNC_S2_INIT = 1 -end - -# select nights in state rsync_s2.new -# output: rsync_s2.book -task nights.rsync_s2.load - host local - periods -poll $TPOLL - periods -exec $TEXEC - periods -timeout 30 - - stdout $LOGDIR/rsync_s2.load.stdout.log - stderr $LOGDIR/rsync_s2.load.stderr.log - - # only run one of these tasks at a time - npending 1 - - task.exec - # command check.rawfix.sh state ckchip.done - command check.rawfix.sh state rsync_s2.new - end - - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s2.book - book delpage rsync_s2.book -key pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -###########################################################3333333 - -task nights.rsync_s2.run - - periods -poll $RPOLL - periods -exec $REXEC - periods -timeout 60 - npending 1 - - stdout $LOGDIR/rsync_s2.run.stdout.log - stderr $LOGDIR/rsync_s2.run.stderr.log - - task.exec - # if we are unable to run the 'exec', use a long retry time - periods -exec $REXEC - - book npages rsync_s2.book -var N - if ($N == 0) break - if ($NETWORK == 0) break - - # look for new nights in rsync_s2.book (pantaskState == INIT) - book getpage rsync_s2.book 0 -var pageName -key pantaskState INIT - if ("$pageName" == "NULL") break - - # echo "got a valid entry: $pageName" - # book listpage rsync_s2.book $pageName - - book setword rsync_s2.book $pageName pantaskState RUN - book getword rsync_s2.book $pageName dateobs -var DATEOBS - option $pageName - - # send output to a file in the DATEOBS directory - stdout $DATEOBS/rsync_s2.log - stderr $DATEOBS/rsync_s2.log - - command rsync -auv $TGTDIR/$DATEOBS/ $SRCDIR/$DATEOBS/ - end - - ## do not delete the page from the book if done here - ## deletion has to happen after the .load - task.exit 0 - echo "--- stdout : nights.rsync_s2.run ---" - queueprint stdout - exec update.rawfix.sh $options:0 rsync_s2.done - book setword rsync_s2.book $options:0 pantaskState DONE - end - - # exit values other than 0: - task.exit default - showcommand failure - book setword rsync_s2.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 rsync_s2.fail - end - - task.exit crash - showcommand crash - book setword rsync_s2.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 rsync_s2.fail - end - - # operation times out? - task.exit timeout - showcommand timeout - book setword rsync_s2.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 rsync_s2.fail - end -end -