Index: /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42414) +++ /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42415) @@ -67,16 +67,29 @@ if ($mystate == "fixchip.done") goto good_state if ($mystate == "fixchip.fail") goto good_state - if ($mystate == "rsync_s1.new") goto good_state - if ($mystate == "rsync_s1.done") goto good_state - if ($mystate == "rsync_s1.fail") goto good_state if ($mystate == "md5sum.new") goto good_state if ($mystate == "md5sum.done") goto good_state if ($mystate == "md5sum.fail") goto good_state - if ($mystate == "rsync_s2.new") goto good_state - if ($mystate == "rsync_s2.done") goto good_state - if ($mystate == "rsync_s2.fail") goto good_state if ($mystate == "md5chk.new") goto good_state if ($mystate == "md5chk.done") goto good_state if ($mystate == "md5chk.fail") goto good_state + + # stage 2: + if ($mystate == "fixchip_s2.new") goto good_state + if ($mystate == "fixchip_s2.done") goto good_state + if ($mystate == "fixchip_s2.fail") goto good_state + if ($mystate == "md5sum_s2.new") goto good_state + if ($mystate == "md5sum_s2.done") goto good_state + if ($mystate == "md5sum_s2.fail") goto good_state + if ($mystate == "md5chk_s2.new") goto good_state + if ($mystate == "md5chk_s2.done") goto good_state + if ($mystate == "md5chk_s2.fail") goto good_state + +# disable the rsyncs stages: run in directory from b-node +# if ($mystate == "rsync_s1.new") goto good_state +# if ($mystate == "rsync_s1.done") goto good_state +# if ($mystate == "rsync_s1.fail") goto good_state +# if ($mystate == "rsync_s2.new") goto good_state +# if ($mystate == "rsync_s2.done") goto good_state +# if ($mystate == "rsync_s2.fail") goto good_state echo "ERROR: unknown mode $mystate" Index: /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42414) +++ /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42415) @@ -67,5 +67,5 @@ my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum); - &check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead); + &check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Ndead); } @@ -249,61 +249,80 @@ my $outfh = $_[0]; my $extID = $_[1]; - my $Nmain = $_[2]; - my $Nback = $_[3]; - my $Ndead = $_[4]; - - # SKIP : already fixed + my $data_st = $_[2]; + my $md5_sum = $_[3]; + my $Nmain = $_[4]; + my $Nback = $_[5]; + my $Ndead = $_[6]; + + my $outinfo = "$extID $data_st $md5_sum"; + + # GOOD : already fixed if (($Nmain == 1) && ($Nback == 1)) { - print $outfh "GOOD: $extID\n"; - return; - } - - # single copy @ ITC, no backup + print $outfh "GOOD: $outinfo\n"; + return; + } + + # SING : single copy @ ITC, no backup # make a backup if (($Nmain == 1) && ($Nback == 0)) { - print $outfh "SING: $extID\n"; - return; - } - - # ITC files with a single backup copies need to be culled - # cull Nmain - 1 - if (($Nmain > 1) && ($Nback == 1)) { - print $outfh "CULL: $extID\n"; - return; - } - - # multiple copies @ ITC + print $outfh "SING: $outinfo\n"; + return; + } + + # SBCK : single copy @ ATRC, no main copy + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print $outfh "SBCK: $outinfo\n"; + return; + } + + # DUPS : multiple copies @ ITC, none at ATRC # make a backup and cull Nmain - 1 if (($Nmain > 1) && ($Nback == 0)) { - print $outfh "DUPS: $extID\n"; - return; - } - - # no more valid instances - if (($Nmain == 0) && ($Nback == 0) && $Ndead) { - print $outfh "DEAD: $extID\n"; - return; - } - if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { - print $outfh "GONE: $extID\n"; - return; - } - - # single copy @ ATRC - # make a backup @ ITC - if (($Nmain == 0) && ($Nback == 1)) { - print $outfh "SBCK: $extID\n"; - return; - } - - # multiple copies @ ATRC, but none at ITC + print $outfh "DUPS: $outinfo\n"; + return; + } + + # XBCK : multiple copies @ ATRC, but none at ITC # make a backup at ITC, cull ATRC extras if (($Nmain == 0) && ($Nback > 1)) { - print $outfh "XBCK: $extID\n"; + print $outfh "XBCK: $outinfo\n"; + return; + } + + # DEAD : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print $outfh "DEAD: $outinfo\n"; + return; + } + # GONE : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print $outfh "GONE: $outinfo\n"; + return; + } + + # CUL1 : multiple ITC files, single backup + # cull only ITC + if (($Nmain > 1) && ($Nback == 1)) { + print $outfh "CUL1: $outinfo\n"; + return; + } + + # CUL2 : multiple ATRC files, single ITC + # cull only ATRC + if (($Nback > 1) && ($Nmain == 1)) { + print $outfh "CUL2: $outinfo\n"; + return; + } + + # CUL3 : excess ATRC & ITC files, cull from both? + # cull Nmain - 1 + if (($Nback > 1) && ($Nmain > 1)) { + print $outfh "CUL3: $outinfo\n"; return; } # unexpected state: - print $outfh "OOPS: $extID\n"; + print $outfh "OOPS: $outinfo\n"; return; } Index: /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (revision 42414) +++ /trunk/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (revision 42415) @@ -1,5 +1,5 @@ #! /usr/bin/env perl -my $DEBUG = 4; +my $DEBUG = 1; my $time = time(); @@ -82,4 +82,8 @@ print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; $Nfail ++; + } else { + if ($DEBUG > 2) { + print STDOUT "$inst $state $md5tru = $md5obs\n"; + } } } Index: /trunk/tools/eam/rawfix.20230221/src/check_night_status.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check_night_status.pl (revision 42415) +++ /trunk/tools/eam/rawfix.20230221/src/check_night_status.pl (revision 42415) @@ -0,0 +1,131 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_night_status.pl --dateobs (night) --chiplist (file) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$dateword/$chip_id.uris.txt"; + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist); +} + Index: /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42414) +++ /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42415) @@ -36,4 +36,8 @@ my $Nxbck = 0; +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + foreach my $chip_id (@chip_ids) { @@ -49,15 +53,15 @@ # we expect lines of the form: STATE PATH - # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits - if (@words != 2) { die "error in URI list $urifile, line: $line\n"; } + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } my $state = $words[0]; my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; if ($state eq "GOOD:") { $Ngood++; next; } if ($state eq "DEAD:") { $Ndead++; next; } if ($state eq "GONE:") { $Ngone++; next; } - if ($state eq "DUPS:") { $Ndups++; next; } - if ($state eq "XBCK:") { $Nxbck++; next; } # one copy at ITC: neb-replicate will create a copy at ATRC @@ -68,5 +72,5 @@ unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } open (OUTFILE, ">>$dateword/$newhost.inst.txt"); - print OUTFILE "$newfile\n"; + print OUTFILE "$newfile $datast $md5sum\n"; close (OUTFILE); $Nsing ++; @@ -82,9 +86,18 @@ unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } open (OUTFILE, ">>$dateword/$newhost.inst.txt"); - print OUTFILE "$newfile\n"; + print OUTFILE "$newfile $datast $md5sum\n"; close (OUTFILE); $Nsbck ++; next; } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } } $volnum ++; @@ -98,4 +111,7 @@ print "Nsbck: $Nsbck\n"; print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; exit 0; Index: /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage2.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage2.pl (revision 42415) +++ /trunk/tools/eam/rawfix.20230221/src/fix_chip_locations_stage2.pl (revision 42415) @@ -0,0 +1,251 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ('nebulous.config'); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# choose ITC machines on which to place backup copies: +my @volarray = &get_neb_volumes; +my $volnum = 0; + +my $Ngood = 0; +my $Ndead = 0; +my $Ngone = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$dateword/$chip_id.uris.txt"; + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these were done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } # one copy at ITC: should not happen at stage 2 + if ($state eq "SBCK:") { $Nsbck++; next; } # one copy at ATRC: should not happen at stage 2 + + # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies + if ($state eq "DUPS:") { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$dateword/$newhost.inst.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Ndups++; + next; + } + + # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume + if ($state eq "XBCK:") { + my $tgtvol = $volarray[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$dateword/$newhost.inst.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nxbck++; + next; + } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + $volnum ++; +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_new_instance { + + my $ext_id = $_[0]; + + ## the first instance should be the last one created: + my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $uri = $values[0]; + my $host = $values[1]; + my $xattr = $values[2]; + + return ($uri, $host, $xattr); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my @volarray; + + my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@volarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @volarray; +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances_stage2.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances_stage2.pl (revision 42415) +++ /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances_stage2.pl (revision 42415) @@ -0,0 +1,133 @@ +#!/usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) + +my ($myhost, $input) = &parse_cmdopts; + +# check that we are running this script on 'myhost' +my $truhost = `hostname -s`; chomp $truhost; +if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } + +# read the list of instances: +my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); + +my $output = $input; +$output =~ s/inst/md5s/; + +if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } + +# read in both instance +my $out_hash_r; +if (-e $output) { + $out_hash_r = &load_output_list ($output); +} + +open (OUTFILE, ">>$output"); + +# foreach my $instance (@$ins_list_r) { +for (my $i = 0; $i < @$ins_list_r; $i ++) { + + my $instance = $ins_list_r->[$i]; + my $data_state = $dst_list_r->[$i]; + my $md5sum_tru = $md5_list_r->[$i]; + + # if the instance is already in the output file, skip it + if (defined $out_hash_r) { + my $name = $out_hash_r->{$instance}; + if (defined $name) { + print STDERR "skipping $instance (already in output)\n"; + next; + } + } + + my $filename = $instance; + $filename =~ s|file://||; + + my $line; + if (0) { + $line = "fakevalue $filename"; # XXX TEST: disable calculation: + } else { + $line = `md5sum $filename`; + } + + my ($md5sum, $rawfile) = split (" ", $line); + print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n"; +} +close OUTFILE; +exit 0; + +# read the instance info +sub load_instance_list { + my $infile = $_[0]; + + my (@ins_list, @dst_list, @md5_list); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 3) { die "error in input $infile : $line\n"; } + push (@ins_list, $words[0]); + push (@dst_list, $words[1]); + push (@md5_list, $words[2]); + } + close (FILE); + + return (\@ins_list, \@dst_list, \@md5_list); +} + +# read the instance list in the output file +sub load_output_list { + my $infile = $_[0]; + + my (%out_hash); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 4) { die "error in input $infile : $line\n"; } + $out_hash{$words[0]} = 1; + } + close (FILE); + + return (\%out_hash); +} + +sub parse_cmdopts { + + my ($myhost, $input); + GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input) || pod2usage(2); + + pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + + return ($myhost, $input); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt (revision 42415) +++ /trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip_s2.pt (revision 42415) @@ -0,0 +1,119 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_S2_INIT)) + book init fixchip_s2.book + $RAWFIX_FIXCHIP_S2_INIT = 1 +end + +# select nights in state fixchip_s2.new +# output: fixchip_s2.book +task nights.fixchip_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip_s2.load.stdout.log + stderr $LOGDIR/fixchip_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s2.book + book delpage fixchip_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip_s2.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/fixchip_s2.run.stdout.log + stderr $LOGDIR/fixchip_s2.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip_s2.book (pantaskState == INIT) + book getpage fixchip_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip_s2.book $pageName pantaskState RUN + book getword fixchip_s2.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command fix_chip_locations_stage2.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword fixchip_s2.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip_s2.done + book delpage fixchip_s2.book $options:0 + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip_s2.fail + end + + task.exit crash + showcommand crash + book setword fixchip_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip_s2.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk_s2.pt (revision 42415) +++ /trunk/tools/eam/rawfix.20230221/src/rawfix.md5chk_s2.pt (revision 42415) @@ -0,0 +1,120 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD + +if (not($?RAWFIX_MD5CHK_S2_INIT)) + book init md5chk_s2.book + $RAWFIX_MD5CHK_S2_INIT = 1 +end + +# select nights in state md5chk_s2.new +# output: md5chk_s2.book +task nights.md5chk_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5chk_s2.load.stdout.log + stderr $LOGDIR/md5chk_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state md5chk_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk_s2.book + book delpage md5chk_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.md5chk_s2.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5chk_s2.run.stdout.log + stderr $LOGDIR/md5chk_s2.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5chk_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5chk_s2.book (pantaskState == INIT) + book getpage md5chk_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5chk_s2.book $pageName pantaskState RUN + book getword md5chk_s2.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # send output to a file in the DATEOBS directory + stdout $DATEOBS/md5chk_s2.log + + command check_md5s_dateobs.pl --dateobs $FULLDATE + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword md5chk_s2.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 md5chk_s2.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5chk_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 md5chk_s2.fail + end + + task.exit crash + showcommand crash + book setword md5chk_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 md5chk_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5chk_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 md5chk_s2.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt (revision 42415) +++ /trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum_s2.pt (revision 42415) @@ -0,0 +1,282 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum_s2 calculations +## use the top-level rawfix.pt to load + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_S2_INIT)) + book init md5sum_s2.book + book init md5host_s2.book + $RAWFIX_MD5SUM_S2_INIT = 1 +end + +# select nights which are ready for md5sum_s2 calcs +# output: md5sum_s2.book +task nights.md5sum_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5sum_s2.load.stdout.log + stderr $LOGDIR/md5sum_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state md5sum_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum_s2.book + book delpage md5sum_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.loadhosts + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5sum_s2.loadhosts.stdout.log + stderr $LOGDIR/md5sum_s2.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5sum_s2.book (pantaskState == INIT) + book getpage md5sum_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5sum_s2.book $pageName pantaskState RUN + book getword md5sum_s2.book $pageName dateobs -var DATEOBS + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/md5sum_s2.runhosts.log" $mypath $DATEOBS + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + command get_hosts_md5s.sh $DATEOBS + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host_s2.book + book setword md5sum_s2.book $options:0 NHOST {$Nhosts - 1} + book setword md5sum_s2.book $options:0 NDONE 0 + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum_s2.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum_s2.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.runhosts + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + + stdout $LOGDIR/md5sum_s2.runhosts.stdout.log + stderr $LOGDIR/md5sum_s2.runhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5host_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in md5host_s2.book (pantaskState == INIT) + book getpage md5host_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5host_s2.book $pageName pantaskState RUN + book getword md5host_s2.book $pageName dateobs -var DATEOBS + book getword md5host_s2.book $pageName md5host -var MD5HOST + + option $pageName $DATEOBS + # save DATEOBS so we can find the md5sum_s2.book entry + + # XXX TEST: $USE_HOST = ipp060 + + $USE_HOST = $MD5HOST + host -required $USE_HOST + ## check if this host needs to be launched + if (not($?host:$USE_HOST)) + echo "host not defined, creating : $USE_HOST" + $host:$USE_HOST = 0 + end + if (not($host:$USE_HOST)) + control host add $USE_HOST + $host:$USE_HOST = 1 + end + + ## even when testing, use the original hostname here: + $input = $mypath/$DATEOBS/$MD5HOST.inst.txt + + # need to make sure this will work on the ipp b-nodes: + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command $mypath/get_md5s_instances_stage2.pl --hostname $USE_HOST --input $input + end + + # dump stdout / stderr to a file on failure? + task.exit 0 + nights.mdsum.exit OK + end + + # exit values other than 0: + task.exit default + nights.mdsum.exit FAIL + showcommand failure + end + + task.exit crash + nights.mdsum.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.mdsum.exit TIMEOUT + showcommand timeout + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5sum_s2.clean.stdout.log + stderr $LOGDIR/md5sum_s2.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in md5sum_s2.book (pantaskState == DONE) + book getpage md5sum_s2.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage md5sum_s2.book $pageName + + # book setword md5sum_s2.book $pageName pantaskState RUN + book getword md5sum_s2.book $pageName dateobs -var DATEOBS + book getword md5sum_s2.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS md5sum_s2.done + else + command update.rawfix.sh $DATEOBS md5sum_s2.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword md5sum_s2.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum_s2.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum_s2.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.mdsum.exit + if ($0 != 2) + echo "USAGE: nights.mdsum.exit (quality)" + break + end + + book setword md5host_s2.book $options:0 pantaskState DONE + book getword md5sum_s2.book $options:1 NHOST -var Nhosts + book getword md5sum_s2.book $options:1 NDONE -var Ndone + $Ndone ++ + book setword md5sum_s2.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword md5sum_s2.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/md5sum_s2.runhosts.log" $mypath $options:1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword md5sum_s2.book $options:1 pantaskState STOP + end + book delpage md5host_s2.book $options:0 +end + Index: /trunk/tools/eam/rawfix.20230221/src/rawfix.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/rawfix.pt (revision 42414) +++ /trunk/tools/eam/rawfix.20230221/src/rawfix.pt (revision 42415) @@ -9,5 +9,9 @@ # use this to find the files in this local directory -$mypath = /home/panstarrs/eugene/rawfix.20230221 +# $mypath = /home/panstarrs/eugene/rawfix.20230221 + +# NOTE: use a directory located on a b-node so the results can +# be seen by both ATRC and ITC machines +$mypath = /data/ippb07.0/home/eugene/rawfix.20230221 if (not($?LOGDIR)) @@ -19,10 +23,15 @@ # order is this file is not determinative, only the # change of state from STAGE1.new -> STAGE1.done -> STAGE2.new -input rawfix.ckchip.pt -input rawfix.fixchip.pt -input rawfix.rsync_s1.pt -input rawfix.md5sum.pt -input rawfix.rsync_s2.pt -input rawfix.md5chk.pt +# input rawfix.ckchip.pt +# input rawfix.fixchip.pt +# input rawfix.md5sum.pt +# input rawfix.md5chk.pt +# input rawfix.fixchip_s2.pt +input rawfix.md5sum_s2.pt +#input rawfix.md5chk_s2.pt + +# no need to run rsync if we use the b-node working directory: +# input rawfix.rsync_s1.pt +# input rawfix.rsync_s2.pt ######################### UTILITIES #################################