Index: /trunk/tools/eam/rawfix.20230221/src.v0/advance.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/advance.rawfix.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/advance.rawfix.sh (revision 42443) @@ -0,0 +1,94 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: advance.rawfix.sh all (Nmax)" + echo "USAGE: advance.rawfix.sh night (YYYYMMDD)" + exit 2 +endif + +set mode = $1 + +## include a max number of active programs +if ($mode == "all") then + if ($#argv != 2) then + echo "USAGE: advance.rawfix.sh all (Nmax)" + exit 2 + endif + + set nMax = $2 + set nActive = 0 + set nAdvance = 0 + set nights = `./check.rawfix.sh all | awk '(NR > 1){print $1}'` + foreach f ($nights) + ./advance.rawfix.sh night $f + set myResult = $status + if ($myResult == 1) continue + if ($myResult == 2) continue + if ($myResult == 3) continue + if ($myResult == 4) then + @ nActive ++ + endif + if ($myResult == 5) then + @ nActive ++ + @ nAdvance ++ + endif + if ($nActive >= $nMax) then + echo "advance $nAdvance nights, total of $nActive active nights" + exit 0 + endif + end + echo "advance $nAdvance nights, total of $nActive active nights" + exit 0 +endif + +# advance STAGE0.done to STAGE1.new +# if already in STAGEx.new, return 4 +# if setting to STAGEx.new, return 5 +# otherwise, return 3 +if ($mode == "night") then + if ($#argv != 2) then + echo "USAGE: advance.rawfix.sh night (YYYYMMDD)" + exit 2 + endif + + set night = $2 + + set mystate = `./check.rawfix.sh night $2 | awk '(NR == 2){print $2}'` + + set tgtstate = "NONE" + if ($mystate == "new") set tgtstate = "ckchip.new" + if ($mystate == "ckchip.done") set tgtstate = "fixchip.new" + if ($mystate == "fixchip.done") set tgtstate = "md5sum.new" + if ($mystate == "md5sum.done") set tgtstate = "md5chk.new" + if ($mystate == "md5chk.done") set tgtstate = "fixchip_s2.new" + if ($mystate == "fixchip_s2.done") set tgtstate = "md5sum_s2.new" + if ($mystate == "md5sum_s2.done") set tgtstate = "md5chk_s2.new" + if ($mystate == "md5chk_s2.done") set tgtstate = "fixchip_s3.new" + if ($mystate == "fixchip_s3.done") set tgtstate = "ckchip_s3.new" + if ($mystate == "ckchip_s3.done") set tgtstate = "DONE" + + if ($mystate == "ckchip.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip.new") set tgtstate = "ACTIVE" + if ($mystate == "md5sum.new") set tgtstate = "ACTIVE" + if ($mystate == "md5chk.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "md5sum_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "md5chk_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip_s3.new") set tgtstate = "ACTIVE" + if ($mystate == "ckchip_s3.new") set tgtstate = "ACTIVE" + + echo "$night : $mystate : $tgtstate" + + # do not set these two states + if ($tgtstate == "NONE") exit 3 + if ($tgtstate == "ACTIVE") exit 4 + + # only set these states + ./update.rawfix.sh $night $tgtstate + + if ($tgtstate == "DONE") exit 1 + exit 5 +endif + +echo "ERROR: unknown mode: $mode" +exit 2 Index: /trunk/tools/eam/rawfix.20230221/src.v0/check.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/check.rawfix.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/check.rawfix.sh (revision 42443) @@ -0,0 +1,115 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: check.rawfix.sh (mode) [options]" + echo "USAGE: check.rawfix.sh all" + echo "USAGE: check.rawfix.sh stats" + echo "USAGE: check.rawfix.sh night (YYYYMMDD)" + echo "USAGE: check.rawfix.sh state (state)" + echo " NOTE: state = raw, rsync, md5sum.new" + exit 2 +endif + +set mode = $1 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" +set DBOPTS = "-B -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" + +if ($mode == "all") then + if ($#argv != 1) then + echo "USAGE: check.rawfix.sh all" + exit 2 + endif + + mysql $DBOPTS -e "select dateobs, state from nights" + exit 0 +endif + +if ($mode == "stats") then + if ($#argv != 1) then + echo "USAGE: check.rawfix.sh stats" + exit 2 + endif + + mysql $DBOPTS -e "select count(dateobs) Nnights, state from nights group by state" + exit 0 +endif + +if ($mode == "night") then + if ($#argv != 2) then + echo "USAGE: check.rawfix.sh night (YYYYMMDD)" + exit 2 + endif + + set mynight = $2 + mysql $DBOPTS -e "select dateobs, state from nights where (dateobs = '$mynight')" + exit 0 +endif + +if ($mode == "state") then + if ($#argv != 2) then + echo "USAGE: check.rawfix.sh state (state)" + exit 2 + endif + + set mystate = $2 + + if ($mystate == "ckchip.new") goto good_state + if ($mystate == "ckchip.done") goto good_state + if ($mystate == "ckchip.fail") goto good_state + if ($mystate == "fixchip.new") goto good_state + if ($mystate == "fixchip.done") goto good_state + if ($mystate == "fixchip.fail") goto good_state + if ($mystate == "md5sum.new") goto good_state + if ($mystate == "md5sum.done") goto good_state + if ($mystate == "md5sum.fail") goto good_state + if ($mystate == "md5chk.new") goto good_state + if ($mystate == "md5chk.done") goto good_state + if ($mystate == "md5chk.fail") goto good_state + + # stage 2: + if ($mystate == "fixchip_s2.new") goto good_state + if ($mystate == "fixchip_s2.done") goto good_state + if ($mystate == "fixchip_s2.fail") goto good_state + if ($mystate == "md5sum_s2.new") goto good_state + if ($mystate == "md5sum_s2.done") goto good_state + if ($mystate == "md5sum_s2.fail") goto good_state + if ($mystate == "md5chk_s2.new") goto good_state + if ($mystate == "md5chk_s2.done") goto good_state + if ($mystate == "md5chk_s2.fail") goto good_state + + # stage 3: + if ($mystate == "fixchip_s3.new") goto good_state + if ($mystate == "fixchip_s3.done") goto good_state + if ($mystate == "fixchip_s3.fail") goto good_state + if ($mystate == "ckchip_s3.new") goto good_state + if ($mystate == "ckchip_s3.done") goto good_state + if ($mystate == "ckchip_s3.fail") goto good_state + +# disable the rsyncs stages: run in directory from b-node +# if ($mystate == "rsync_s1.new") goto good_state +# if ($mystate == "rsync_s1.done") goto good_state +# if ($mystate == "rsync_s1.fail") goto good_state +# if ($mystate == "rsync_s2.new") goto good_state +# if ($mystate == "rsync_s2.done") goto good_state +# if ($mystate == "rsync_s2.fail") goto good_state + + echo "ERROR: unknown mode $mystate" + exit 2 + + good_state: + set t = `date +%T..%N` + mysql $DBOPTS -e "select dateobs, curtime() as timestamp from nights where (state = '$mystate')" | awk -v T=$t '(NR == 1){print $0}(NR > 1){print $1, T}' + exit 0 +endif + +echo "ERROR: unknown mode $mode" +exit 2 + + Index: /trunk/tools/eam/rawfix.20230221/src.v0/check_chip_locations.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/check_chip_locations.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/check_chip_locations.pl (revision 42443) @@ -0,0 +1,423 @@ +#! /usr/bin/env perl + +# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases +my $DEBUG = 3; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) +# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage v0 + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($neb_dbh, $gpc_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# a global to store the number of instances per host (for each chip) +my %hostcount; +my %hostfile; + +# put output files in a directory defined by the night +mkdir "$topdir/$dateword"; + +# output files: +# instance lists by host: YYYYMMDD/HOSTNAME.inst.STAGE.txt +# chip states by chip: YYYYMMDD/CHIPNAME.uris.txt + +foreach my $chip_id (@chip_ids) { + + my $outfh; + open ($outfh, ">$topdir/$dateword/$chip_id.uris.txt"); + + # get the list of chip neb paths for one chip + my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id); + + # I think this makes a copy of the arrays, which is inefficient + # my @neb_paths = @$neb_paths_r; + # my @ds_array = @$ds_array_r; + # my @md_array = @$md_array_r; + + # reset the hostcount for this chip + %hostcount = (); + + # find the neb instances for each chip nebulous entry + + for (my $i = 0; $i < @$neb_paths_r; $i++) { + my $nebname = $neb_paths_r->[$i]; # equivalent to ${$neb_paths_r}[$i] + my $data_st = $ds_array_r->[$i]; + my $md5_sum = $md_array_r->[$i]; + + # print STDERR "nebname: $nebname, dstate: $data_st, md5: $md5_sum\n"; + + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($nebname); + + # determine the status of the instances + my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum); + + &check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Ndead); + } + + close ($outfh); + + # now save the host count for this chip + my $hostfh; + open ($hostfh, ">$topdir/$dateword/$chip_id.hosts.txt"); + + # save the host count for this chip + my @hostnames = keys %hostcount; + foreach my $myname (@hostnames) { + print $hostfh "$myname $hostcount{$myname}\n"; + } + close ($hostfh); +} + +# save the host count for this chip +my @hostnames = keys %hostfile; +foreach my $myname (@hostnames) { + close ($hostfile{$myname}); + if ($DEBUG > 4) { print "close host instance file for $myname\n"; } +} + +# run neb-replicate for SING entries? +# run neb-replicate for SBCK entries? (does this put a second copy at ATRC or ITC?) +# save the list of instances on each host +## run md5sum on the instances for each host +## only note inconsistent md5sums? (need to save the gpc1 md5sum values) +# rerun to generate a list of the host with max instances per chip? + +### when to cull entries? +## neb-replicate puts the new entry at ATRC by default +## neb-replicate --volume VOL puts the new entry at the specified location +### in order to use this, I need to choose the target volume + +exit 0; + +# die "done"; + +sub vsystem { + # print STDERR "@_\n"; + return 0; + #my $status = system ("@_"); + #return $status; +} + + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +# get the list of chip neb paths for one chip +sub get_chip_paths { + my $fulldate = $_[0]; + my $chip_id = $_[1]; + + my (@nebarray, @ds_array, @md_array); + + my $sql_chip_paths = "SELECT uri, data_state, md5sum from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'"; + if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } + + my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows chip images\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only uri selected above) + my $Nvalues = @values; + if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; } + + push (@nebarray, $values[0]); + push (@ds_array, $values[1]); + push (@md_array, $values[2]); + if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; } + } + + # return references so we can keep the three arrays distinct + return (\@nebarray, \@ds_array, \@md_array); +} + +sub get_chip_instances { + + my $nebname = $_[0]; + + # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + + # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + if ($DEBUG > 5) { print "nebname: $ext_id\n"; } + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# count the number of instances at ITC (0), ATRC (1), or other locations (>1) +# record the instances in a file for each host +# count the number of instances on each host +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + my $data_st = $_[2]; + my $md5_sum = $_[3]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + my $Ndead = 0; + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + # the number of returned values per row must match the query above (in get_chip_instances) + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if ($xattr > 1) { $Ndead ++; next; } # do not save instances on dead partitions + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + + # count the number of each host by incrementing the hash element corresponding to the hostname: + # NOTE: hostcount is global + $hostcount{$hostname} ++; + + if (not defined $hostfile{$hostname}) { + open ($hostfile{$hostname}, ">$topdir/$dateword/$hostname.inst.$stage.txt"); + ## XXX deal with open failure here + } + + my $fh = $hostfile{$hostname}; + print $fh "$instance $data_st $md5_sum\n"; + } + + return ($Nmain, $Nback, $Ndead); +} + +sub check_instance_state { + + my $outfh = $_[0]; + my $extID = $_[1]; + my $data_st = $_[2]; + my $md5_sum = $_[3]; + my $Nmain = $_[4]; + my $Nback = $_[5]; + my $Ndead = $_[6]; + + my $outinfo = "$extID $data_st $md5_sum"; + + # GOOD : already fixed + if (($Nmain == 1) && ($Nback == 1)) { + print $outfh "GOOD: $outinfo\n"; + return; + } + + # SING : single copy @ ITC, no backup + # make a backup + if (($Nmain == 1) && ($Nback == 0)) { + print $outfh "SING: $outinfo\n"; + return; + } + + # SBCK : single copy @ ATRC, no main copy + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print $outfh "SBCK: $outinfo\n"; + return; + } + + # DUPS : multiple copies @ ITC, none at ATRC + # make a backup and cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 0)) { + print $outfh "DUPS: $outinfo\n"; + return; + } + + # XBCK : multiple copies @ ATRC, but none at ITC + # make a backup at ITC, cull ATRC extras + if (($Nmain == 0) && ($Nback > 1)) { + print $outfh "XBCK: $outinfo\n"; + return; + } + + # DEAD : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print $outfh "DEAD: $outinfo\n"; + return; + } + # GONE : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print $outfh "GONE: $outinfo\n"; + return; + } + + # CUL1 : multiple ITC files, single backup + # cull only ITC + if (($Nmain > 1) && ($Nback == 1)) { + print $outfh "CUL1: $outinfo\n"; + return; + } + + # CUL2 : multiple ATRC files, single ITC + # cull only ATRC + if (($Nback > 1) && ($Nmain == 1)) { + print $outfh "CUL2: $outinfo\n"; + return; + } + + # CUL3 : excess ATRC & ITC files, cull from both? + # cull Nmain - 1 + if (($Nback > 1) && ($Nmain > 1)) { + print $outfh "CUL3: $outinfo\n"; + return; + } + + # unexpected state: + print $outfh "OOPS: $outinfo\n"; + return; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir) +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; + pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; + pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; + pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; + pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; + if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } + my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh, $gpc_dbh); +} + + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /trunk/tools/eam/rawfix.20230221/src.v0/check_chip_locations.v0.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/check_chip_locations.v0.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/check_chip_locations.v0.pl (revision 42443) @@ -0,0 +1,423 @@ +#! /usr/bin/env perl + +# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases + +# TESTRUN can be 1, 2, 3, 4 to get further in the tests +my $TESTRUN = 2; +my $DEBUG = 3; + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($tgthost,$tgtvol,$subdir,$movelist); + +# USAGE: check_chip_locations.pl --dateobs (night) --chip_id (chip_id) +# USAGE: check_chip_locations.pl --dateobs 2010.05.01 --chip_id XY33 + +my ($dateobs, $chip_id); +GetOptions( 'dateobs=s' => \$dateobs, 'chip_id=s' => \$chip_id) || pod2usage(2); + +my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); +my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); +my $conffile = 'nebulous.config'; + +# parse the nebulous.config file: +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; +pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; +pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; +pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; +pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + +pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; +pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; +pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; +pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; +pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + +pod2usage( -msg => "Cannot determine target date", -exitval => 2) unless defined $dateobs; +pod2usage( -msg => "Cannot determine target chip", -exitval => 2) unless defined $chip_id; + +# check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + +my ($date_year, $date_month, $date_day) = split "/",$dateobs; +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + +my $time = time(); + +my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; +if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } +my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; +if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } +my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# get the list of chip neb paths: +my @nebarray; +if (1) { + my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$date_year/$date_month/$date_day,00:00:00' and dateobs < '$date_year/$date_month/$date_day,23:59:59'"; + if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } + + my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows chip images\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only uri selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@nebarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } +} + +# find the neb instances for each chip nebulous entry + +my %hostcount; + +foreach my $nebname (@nebarray) { + + # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + if ($DEBUG > 5) { print "nebname: $ext_id\n"; } + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + if (0) { + foreach my $valref (@tmparray) { + my @values = @$valref; + # number of values should match list above (can this even fail?) + my $Nvalues = @values; + if ($Nvalues != 5) { print "ERROR: wrong number of fields?\n"; next; } + + if ($DEBUG > 2) { print "uri: $values[0], host: $values[1], avail: $values[2], xattr: $values[3], cab: $values[4]\n"; } + } + } + + # my $summary = "$ext_id $so_id"; + my $summary = "$ext_id"; + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + my $Ndead = 0; + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $tmparray[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + my $xattr = $values[3]; + my $hostname = $values[1]; + + # count the number of each host by incrementing the hash element corresponding to the hostname: + $hostcount{$hostname} ++; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + if ($xattr > 1) { $Ndead ++; } + } + + # SKIP : already fixed + if (($Nmain == 1) && ($Nback == 1)) { + print STDOUT "GOOD: $summary\n"; + next; + } + + # single copy @ ITC, no backup + # make a backup + if (($Nmain == 1) && ($Nback == 0)) { + print STDOUT "SING: $summary\n"; + my $status = vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } + next; + } + + # ITC files with a single backup copies need to be culled + # cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 1)) { + my $Ncull = $Nmain - 1; + print STDOUT "CULL: $summary\n"; + for (my $j = 0; $j < $Ncull; $j ++) { + my $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } + } + next; + } + + # multiple copies @ ITC + # make a backup and cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 0)) { + my $Ncull = $Nmain - 1; + print STDOUT "DUPS: $summary\n"; + my $status = vsystem("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } + } + next; + } + + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print STDOUT "DEAD: $summary\n"; + next; + } + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print STDOUT "GONE: $summary\n"; + next; + } + + # single copy @ ATRC + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print STDOUT "SBCK: $summary\n"; + next; + } + + # multiple copies @ ATRC, but none at ITC + # make a backup at ITC, cull ATRC extras + if (($Nmain == 0) && ($Nback > 1)) { + print STDOUT "XBCK: $summary\n"; + next; + } + + # unexpected state: + print STDOUT "OOPS: $summary\n"; + next; +} + +my @hostnames = keys %hostcount; +foreach my $myname (@hostnames) { + print "host $myname = $hostcount{$myname}\n"; +} + +die "done"; + +#my $query = $dbh->prepare( $sql_get_volume_id ); + +#$query->execute( $tgt_volume ); +#my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +#$query->finish; + + +### XX +### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; + +# load the list of files from movelist + +##XX open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +##XX my @movefiles = ; +##XX close (MOVELIST); +##XX +##XX my $otime = $time; +##XX $time = time(); +##XX my $dtime = $time - $otime; +##XX +##XX for (my $i = 0; $i < @movefiles; $i++) { +##XX +##XX my $movefile = $movefiles[$i]; +##XX chomp $movefile; +##XX +##XX # the filename should have the following form: +##XX # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: +##XX # NN : 2-hexdigit name of the subsubdir +##XX # DDDDD : instance ID (long) +##XX # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name +##XX +##XX # I want to check two things: +##XX # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile +##XX # 2) does the vol_id of the instance_id match our vol_id ? +##XX +##XX unless (-e "$tgt_subdir/$movefile") { +##XX print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; +##XX next; +##XX } +##XX +##XX # print "movefile: $movefile\n"; +##XX +##XX my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; +##XX my $summary = "$tgt_subdir/$movefile"; +##XX +##XX if ( ! defined ($ins_id)) { +##XX print "ERROR : INVALID ID: $summary\n"; +##XX next; +##XX } +##XX +##XX if ( $ins_id =~ m/\D/ ) { +##XX print "ERROR : INVALID ID: $summary\n"; +##XX next; +##XX } +##XX $summary = "$summary : $ins_id"; +##XX +##XX # get so_id for this ins_id: +##XX my $so_id = 0; +##XX if (1) { +##XX my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $so_id_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # We should find this instance +##XX print "ERROR : NO DB ENTRY: $summary\n"; +##XX next; +##XX } +##XX +##XX if ($Nrows > 1) { # We should only find one instance +##XX print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; +##XX next; +##XX } +##XX +##XX my $valref = $array[0]; +##XX my @values = @$valref; +##XX +##XX my $Nvalue = @values; +##XX +##XX if ($Nvalue != 1) { +##XX print "ERROR : INVALID DB RESPONSE: $summary\n"; +##XX next; +##XX } +##XX $so_id = $values[0]; +##XX } +##XX +##XX $summary = "$summary : $so_id"; +##XX +##XX # get user_copies for this storage_object +##XX my $Ncopies = 0; +##XX my $ext_id = ""; +##XX if (1) { +##XX my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # user.copies is not set, skip +##XX print "SKIP: NO USER.COPIES ENTRY: $summary\n"; +##XX next; +##XX } +##XX +##XX if ($Nrows > 1) { # We should only find one instance +##XX print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; +##XX next; +##XX } +##XX +##XX my $valref = $array[0]; +##XX my @values = @$valref; +##XX +##XX my $Nvalue = @values; +##XX +##XX if ($Nvalue != 2) { +##XX print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; +##XX next; +##XX } +##XX $ext_id = $values[0]; +##XX $Ncopies = $values[1]; +##XX } +##XX +##XX $summary = "$summary : $ext_id"; +##XX +##XX if ($Ncopies < 2) { +##XX print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; +##XX next; +##XX } +##XX +##XX # check for backup instances +##XX if (1) { +##XX my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # We should find this instance +##XX print "REPL: NO BACKUP COPY: $summary\n"; +##XX next; +##XX } +##XX +##XX ## my $valref = $array[0]; +##XX ## my @values = @$valref; +##XX ## +##XX ## my $Nvalue = @values; +##XX ## +##XX ## if ($Nvalue != 1) { +##XX ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; +##XX ## next; +##XX ## } +##XX } +##XX +##XX } # END loop over movefiles + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + +sub vsystem { + # print STDERR "@_\n"; + return 0; + #my $status = system ("@_"); + #return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/check_md5s_dateobs.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/check_md5s_dateobs.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/check_md5s_dateobs.pl (revision 42443) @@ -0,0 +1,127 @@ +#! /usr/bin/env perl + +my $DEBUG = 1; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_md5s_dateobs.pl --dateobs (night) --topdir (topdir) + +my ($dateword, $stage, $topdir) = &parse_cmdopts; + +my @instlist = glob ("$topdir/$dateword/*.inst.$stage.txt"); +my @md5slist = glob ("$topdir/$dateword/*.md5s.txt"); + +my $Nfail = 0; + +foreach my $instfile (@instlist) { + + # find the matching md5sum file for this host + + my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|; + # print STDERR "host: $host\n"; + + my $md5match = "NONE"; + foreach my $md5sfile (@md5slist) { + + my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|; + # print STDERR "md5host: $md5shost\n"; + + if ($host ne $md5shost) { next; } + + $md5match = $md5sfile; + last; + } + + if ($md5match eq "NONE") { + print STDOUT "missing md5sum file for $instfile\n"; + $Nfail ++; + next; + } + + ## read in the md5sums + open (FILE, $md5match); + my @md5data = ; + close (FILE); + + ## read in the list of instances + open (FILE, $instfile); + my @instdata = ; + close (FILE); + + # if these do not match, complain and move on + my $N1 = @md5data; + my $N2 = @instdata; + if ($N1 != $N2) { + print STDOUT "inconsistent number of lines in $md5match : $N1 vs $N2\n"; + $Nfail ++; + next; + } + + # check for mismatched MD5sums + foreach my $line (@md5data) { + + chomp $line; + my @words = split (/\s+/,$line); + + # we expect lines of the form: INST STATE MDtru MDobs + if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; } + + my $inst = $words[0]; + my $state = $words[1]; + my $md5tru = $words[2]; + my $md5obs = $words[3]; + + if ($md5tru ne $md5obs) { + print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; + $Nfail ++; + } else { + if ($DEBUG > 2) { + print STDOUT "$inst $state $md5tru = $md5obs\n"; + } + } + } +} + +if ($Nfail) { + print STDOUT "ERROR: $Nfail errors\n"; + exit 1; +} else { + print STDOUT "SUCCESS: $Nfail errors\n"; + exit 0; +} + +sub parse_cmdopts { + + my ($dateobs, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($dateword, $stage, $topdir); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/check_night_status.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/check_night_status.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/check_night_status.pl (revision 42443) @@ -0,0 +1,131 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_night_status.pl --dateobs (night) --chiplist (file) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$dateword/$chip_id.uris.txt"; + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist); +} + Index: /trunk/tools/eam/rawfix.20230221/src.v0/create.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/create.rawfix.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/create.rawfix.sh (revision 42443) @@ -0,0 +1,14 @@ +#!/bin/csh + +# make the rawfix database to track rawfix operations +set DBNAME = "rawfix" +set DBUSER = "dvo" + +mysql -h localhost -u root < qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_failures.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) +# USAGE: fix_chip_failures.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage v0 + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; + + +my (@instance, @data_state, @md5tru, @md5obs) = &load_failures; + +sub load_failures { + + my (@instance, @data_state, @md5tru, @md5obs); + + # given a night, load the failures from log.md5chk.v0 + open (INLOG, "$topdir/$dateword/log.md5chk.$stage"); + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + unless ($line =~ /^FAIL:/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 5) { die "error in log output $line\n"; } + + push (@instance, $words[1]); + push (@data_state, $words[2]); + push (@md5tru, $words[3]); + push (@md5obs, $words[4]); + } + close (CHIPS); + + return (@instance, @data_state, @md5tru, @md5obs); +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir) +} + +# identify the + +sub neb_reverse { + + my (@neb_path) = split /\//, $file; + my $neb_key = $neb_path[-1]; + # Remove ins_id + $neb_key =~ s/^\d+\.//; + # Convert colons to slashes. + $neb_key =~ s%:%/%g; + $neb_key =~ s%.fz$%%; + print "$neb_key\n"; +} + Index: /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations.pl (revision 42443) @@ -0,0 +1,253 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# choose ITC machines on which to place backup copies: +my @volarray = &get_neb_volumes; +my $volnum = 0; + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; + open (FILE, $urifile); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # one copy at ITC: neb-replicate will create a copy at ATRC + if ($state eq "SING:") { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL SING: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nsing ++; + next; + } + + # one copy at ATRC: neb-replicate will create a copy at ITC iff we supply the target volume + if ($state eq "SBCK:") { + my $tgtvol = $volarray[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL SBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nsbck ++; + next; + } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + $volnum ++; +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_new_instance { + + my $ext_id = $_[0]; + + ## the first instance should be the last one created: + my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $uri = $values[0]; + my $host = $values[1]; + my $xattr = $values[2]; + + return ($uri, $host, $xattr); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my @volarray; + + my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@volarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @volarray; +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage2.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage2.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage2.pl (revision 42443) @@ -0,0 +1,253 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# choose ITC machines on which to place backup copies: +my @volarray = &get_neb_volumes; +my $volnum = 0; + +my $Ngood = 0; +my $Ndead = 0; +my $Ngone = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; + open (FILE, $urifile); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these were done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } # one copy at ITC: should not happen at stage 2 + if ($state eq "SBCK:") { $Nsbck++; next; } # one copy at ATRC: should not happen at stage 2 + + # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies + if ($state eq "DUPS:") { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Ndups++; + next; + } + + # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume + if ($state eq "XBCK:") { + my $tgtvol = $volarray[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nxbck++; + next; + } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + $volnum ++; +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_new_instance { + + my $ext_id = $_[0]; + + ## the first instance should be the last one created: + my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $uri = $values[0]; + my $host = $values[1]; + my $xattr = $values[2]; + + return ($uri, $host, $xattr); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my @volarray; + + my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@volarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @volarray; +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage3.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage3.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage3.pl (revision 42443) @@ -0,0 +1,286 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations_stage3.pl --dateobs (night) --chiplist (file) + +my ($fulldate, $dateword, $chiplist, $topdir) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); +my %volarray = &get_neb_volumes; + +# choose ITC machines on which to place backup copies: +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; + open (FILE, $urifile); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these were replicated in stage 2, so actually match one of the CUL1 or CUL2 entries + # if ($state eq "DUPS:") { $Ndups++; next; } + # if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + # if ($state eq "CUL1:") { $Ncul1++; next; } + # if ($state eq "CUL2:") { $Ncul2++; next; } + # if ($state eq "CUL3:") { $Ncul3++; next; } + + # one copy at ITC: neb-replicate will create a copy at ATRC + if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:") || ($state eq "DUPS:") || ($state eq "XBCK:")) { + + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($ext_id); + + my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id); + + if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n"; next; } + if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; } + + my $NcullTotal = @$cull_list_r; + foreach my $instance (@$cull_list_r) { + my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|; + unless (defined $volarray{$volname}) { die "ERROR: invalid volume name $volname\n"; } + # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n"; + my $status = &vsystem ("neb-cull --volume $volname $ext_id"); + if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; } + } + # print STDERR "Ncull Total $NcullTotal\n"; + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_chip_instances { + + my $ext_id = $_[0]; + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# select 1 instance with xattr = 0, 1 with xattr = 1. +# the rest should be culled. + +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $main_keep = ""; + my $back_keep = ""; + my @cull_list = (); + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + # the number of returned values per row must match the query above (in get_chip_instances) + my $Nvalue = @values; + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; } + if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; } + + # if we make it here, we already have an instance at this location + push (@cull_list, $instance); + } + return ($main_keep, $back_keep, \@cull_list); +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +# +sub parse_cmdopts { + + my ($dateobs, $chiplist, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $topdir); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my %volarray; + + my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + $volarray{$values[0]} = 1; + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return %volarray; +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/get_hosts_md5s.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/get_hosts_md5s.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/get_hosts_md5s.sh (revision 42443) @@ -0,0 +1,24 @@ +#!/bin/csh + +if ($#argv != 3) then + echo "USAGE: get_hosts_md5s.sh (topdir) (dateobs) (stage)" + exit 2 +endif + +set topdir = $1 +set dateobs = $2 +set stage = $3 + +if (! -d $topdir) then + echo "ERROR: missing top-level directory $topdir" + exit 3 +endif + +if (! -d $topdir/$dateobs) then + echo "ERROR: missing dateobs directory $topdir/$dateobs" + exit 3 +endif + +# report both the dateobs and the hostnames +ls $topdir/$dateobs/*.inst.$stage.txt | awk -v dateobs=$dateobs -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $NF}' | sed s/.inst.$stage.txt// +exit 0 Index: /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_dateobs.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_dateobs.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_dateobs.pl (revision 42443) @@ -0,0 +1,50 @@ +#! /usr/bin/env perl + +die "XXX not done yet"; + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_dateobs.pl --dateobs (night) + +my ($fulldate, $dateword) = &parse_cmdopts; + +# first, ensure the dateobs directory is + + + +sub parse_cmdopts { + + my ($dateobs); + GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_instances.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_instances.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_instances.pl (revision 42443) @@ -0,0 +1,107 @@ +#!/usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc) + +my ($myhost, $input, $stage) = &parse_cmdopts; + +# check that we are running this script on 'myhost' +my $truhost = `hostname -s`; chomp $truhost; +if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } + +# read the list of instances: +my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); + +my $output = $input; +$output =~ s/inst.$stage/md5s/; + +if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } +if (-e $output) { + # check that all instances are in the output file? + print "$output exists, skipping\n"; + exit 0; +} + +open (OUTFILE, ">$output"); + +# foreach my $instance (@$ins_list_r) { +for (my $i = 0; $i < @$ins_list_r; $i ++) { + + my $instance = $ins_list_r->[$i]; + my $data_state = $dst_list_r->[$i]; + my $md5sum_tru = $md5_list_r->[$i]; + + my $filename = $instance; + $filename =~ s|file://||; + + # check for the existence of the file first: + unless (-e $filename) { + print OUTFILE "$instance $data_state $md5sum_tru XXX\n"; + next; + } + + my $line; + if (0) { + $line = "fakevalue $filename"; # XXX TEST: disable calculation: + } else { + $line = `md5sum $filename`; + } + + my ($md5sum, $rawfile) = split (" ", $line); + print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n"; +} +close OUTFILE; +exit 0; + +# read the list of chip_ids: +sub load_instance_list { + my $infile = $_[0]; + + my (@ins_list, @dst_list, @md5_list); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 3) { die "error in input $infile : $line\n"; } + push (@ins_list, $words[0]); + push (@dst_list, $words[1]); + push (@md5_list, $words[2]); + } + close (FILE); + + return (\@ins_list, \@dst_list, \@md5_list); +} + +sub parse_cmdopts { + + my ($myhost, $input, $stage); + GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + + return ($myhost, $input, $stage); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_instances_stage2.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_instances_stage2.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/get_md5s_instances_stage2.pl (revision 42443) @@ -0,0 +1,140 @@ +#!/usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc) + +my ($myhost, $input, $stage) = &parse_cmdopts; + +# check that we are running this script on 'myhost' +my $truhost = `hostname -s`; chomp $truhost; +if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } + +# read the list of instances: +my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); + +my $output = $input; +$output =~ s/inst.$stage/md5s/; + +if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } + +# read in both instance +my $out_hash_r; +if (-e $output) { + $out_hash_r = &load_output_list ($output); +} + +open (OUTFILE, ">>$output"); + +# foreach my $instance (@$ins_list_r) { +for (my $i = 0; $i < @$ins_list_r; $i ++) { + + my $instance = $ins_list_r->[$i]; + my $data_state = $dst_list_r->[$i]; + my $md5sum_tru = $md5_list_r->[$i]; + + # if the instance is already in the output file, skip it + if (defined $out_hash_r) { + my $name = $out_hash_r->{$instance}; + if (defined $name) { + print STDERR "skipping $instance (already in output)\n"; + next; + } + } + + my $filename = $instance; + $filename =~ s|file://||; + + # check for the existence of the file first: + unless (-e $filename) { + print OUTFILE "$instance $data_state $md5sum_tru XXX\n"; + next; + } + + my $line; + if (0) { + $line = "fakevalue $filename"; # XXX TEST: disable calculation: + } else { + $line = `md5sum $filename`; + } + + my ($md5sum, $rawfile) = split (" ", $line); + print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n"; +} +close OUTFILE; +exit 0; + +# read the instance info +sub load_instance_list { + my $infile = $_[0]; + + my (@ins_list, @dst_list, @md5_list); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 3) { die "error in input $infile : $line\n"; } + push (@ins_list, $words[0]); + push (@dst_list, $words[1]); + push (@md5_list, $words[2]); + } + close (FILE); + + return (\@ins_list, \@dst_list, \@md5_list); +} + +# read the instance list in the output file +sub load_output_list { + my $infile = $_[0]; + + my (%out_hash); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 4) { die "error in input $infile : $line\n"; } + $out_hash{$words[0]} = 1; + } + close (FILE); + + return (\%out_hash); +} + +sub parse_cmdopts { + + my ($myhost, $input, $stage); + GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + + return ($myhost, $input, $stage); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/gpc1.chips.txt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/gpc1.chips.txt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/gpc1.chips.txt (revision 42443) @@ -0,0 +1,60 @@ +XY01 +XY02 +XY03 +XY04 +XY05 +XY06 +XY10 +XY11 +XY12 +XY13 +XY14 +XY15 +XY16 +XY17 +XY20 +XY21 +XY22 +XY23 +XY24 +XY25 +XY26 +XY27 +XY30 +XY31 +XY32 +XY33 +XY34 +XY35 +XY36 +XY37 +XY40 +XY41 +XY42 +XY43 +XY44 +XY45 +XY46 +XY47 +XY50 +XY51 +XY52 +XY53 +XY54 +XY55 +XY56 +XY57 +XY60 +XY61 +XY62 +XY63 +XY64 +XY65 +XY66 +XY67 +XY71 +XY72 +XY73 +XY74 +XY75 +XY76 Index: /trunk/tools/eam/rawfix.20230221/src.v0/neb_check_location.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/neb_check_location.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/neb_check_location.pl (revision 42443) @@ -0,0 +1,244 @@ +#! /usr/bin/env perl + +# TESTRUN can be 1, 2, 3, 4 to get further in the tests +my $TESTRUN = 2; + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($tgthost,$tgtvol,$subdir,$movelist); + +GetOptions( + 'tgthost=s' => \$tgthost, + 'tgtvol=s' => \$tgtvol, + 'subdir=s' => \$subdir, + 'list=s' => \$movelist) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = 'nebulous.config'; + +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine target machine and volume", -exitval => 2) unless + defined $tgthost && defined $tgtvol; + +pod2usage( -msg => "Cannot determine subdir or list of files which have been moved", -exitval => 2) unless + defined $movelist && defined $subdir; + +my $host_check = `hostname`; chomp($host_check); + +if ($host_check ne $tgthost) { die "Required to run on $tgthost, not on $host_check\n"; } + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# example: +# tgthost : ipp130 +# tgtvol : 0 + +# Get the vol_id of the (local) destination. + +my $tgt_volume = "${tgthost}.${tgtvol}"; +my $tgt_subdir = "/export/${tgthost}.${tgtvol}/nebulous/${subdir}"; + +### XX my $sql_get_volume_id = "SELECT vol_id, name, host, path FROM volume WHERE name = ?"; +### XX my $sql_update_instance = "UPDATE instance SET vol_id = ?, uri = ? WHERE ins_id = ?"; +### XX +### XX my $query = $dbh->prepare( $sql_get_volume_id ); +### XX +### XX $query->execute( $tgt_volume ); +### XX my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +### XX $query->finish; +### XX +### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; + +# load the list of files from movelist + +open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +my @movefiles = ; +close (MOVELIST); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; + +for (my $i = 0; $i < @movefiles; $i++) { + + my $movefile = $movefiles[$i]; + chomp $movefile; + + # the filename should have the following form: + # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: + # NN : 2-hexdigit name of the subsubdir + # DDDDD : instance ID (long) + # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name + + # I want to check two things: + # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile + # 2) does the vol_id of the instance_id match our vol_id ? + + unless (-e "$tgt_subdir/$movefile") { + print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; + next; + } + + # print "movefile: $movefile\n"; + + my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; + my $summary = "$tgt_subdir/$movefile"; + + if ( ! defined ($ins_id)) { + print "ERROR : INVALID ID: $summary\n"; + next; + } + + if ( $ins_id =~ m/\D/ ) { + print "ERROR : INVALID ID: $summary\n"; + next; + } + $summary = "$summary : $ins_id"; + + # get so_id for this ins_id: + my $so_id = 0; + if (1) { + my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; + + my $rowref = $dbh->selectall_arrayref( $so_id_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "ERROR : NO DB ENTRY: $summary\n"; + next; + } + + if ($Nrows > 1) { # We should only find one instance + print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; + next; + } + + my $valref = $array[0]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 1) { + print "ERROR : INVALID DB RESPONSE: $summary\n"; + next; + } + $so_id = $values[0]; + } + + $summary = "$summary : $so_id"; + + # get user_copies for this storage_object + my $Ncopies = 0; + my $ext_id = ""; + if (1) { + my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # user.copies is not set, skip + print "SKIP: NO USER.COPIES ENTRY: $summary\n"; + next; + } + + if ($Nrows > 1) { # We should only find one instance + print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; + next; + } + + my $valref = $array[0]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 2) { + print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + $ext_id = $values[0]; + $Ncopies = $values[1]; + } + + $summary = "$summary : $ext_id"; + + if ($Ncopies < 2) { + print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; + next; + } + + # check for backup instances + if (1) { + my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "REPL: NO BACKUP COPY: $summary\n"; + next; + } + + ## my $valref = $array[0]; + ## my @values = @$valref; + ## + ## my $Nvalue = @values; + ## + ## if ($Nvalue != 1) { + ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + ## next; + ## } + } + +} # END loop over movefiles + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /trunk/tools/eam/rawfix.20230221/src.v0/queue.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/queue.rawfix.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/queue.rawfix.sh (revision 42443) @@ -0,0 +1,41 @@ +#!/bin/csh + +if ($#argv != 2) then + echo "USAGE: queue.rawfix.sh (list) (filename)" + echo "USAGE: queue.rawfix.sh (night) (YYYYMMDD)" + exit 2 +endif + +set mode = $1 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +if ($mode == "list") then + set filename = $2 + + # validate the list: could be DATE STATE or just DATE (prcol 1)? + + # just a big list of dates? + foreach night (`cat $filename`) + queue.rawfix.sh night $night + end + exit 0 +endif + +if ($mode == "night") then + ## XXX validate the format (YYYYMMDD)? + + XXXXX: + + set mynight = $2 + + mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')" + exit 0 +endif + +echo "ERROR: unknown mode $mode" +exit 2 Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.advance.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.advance.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.advance.pt (revision 42443) @@ -0,0 +1,40 @@ + +$MAX_ACTIVE_NIGHTS = 2 + +## regularly run the command advance.rawfits.sh +task nights.advance + periods -poll 10 + periods -exec 10 + periods -timeout 30 + host local + + stdout $LOGDIR/advance.log + stderr $LOGDIR/advance.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # + command advance.rawfix.sh all $MAX_ACTIVE_NIGHTS + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip.pt (revision 42443) @@ -0,0 +1,124 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_CKCHIP_INIT)) + book init ckchip.book + $RAWFIX_CKCHIP_INIT = 1 +end + +# select nights in state ckchip.new +# output: ckchip.book +task nights.ckchip.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/ckchip.load.stdout.log + stderr $LOGDIR/ckchip.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state ckchip.new + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book + book delpage ckchip.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.ckchip.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages ckchip.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in ckchip.book (pantaskState == INIT) + book getpage ckchip.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword ckchip.book $pageName pantaskState RUN + book getword ckchip.book $pageName dateobs -var DATEOBS + option $pageName + + # this is the first place in the proces that this location is known + mkdir $mypath/$DATEOBS + stdout $mypath/$DATEOBS/log.ckchip.runhosts.v0 + stderr $mypath/$DATEOBS/log.ckchip.runhosts.v0 + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command $mypath/check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 + end + + # mark the entry done, will be deleted above in .load + task.exit 0 + book setword ckchip.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 ckchip.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword ckchip.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 ckchip.fail + end + + task.exit crash + showcommand crash + book setword ckchip.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 ckchip.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword ckchip.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 ckchip.fail + end +end + +# option for writing stdout / stderr to a file +# queuepush stdout "------- end of stdout ---------" +# queuesave stdout $options:0/log.ckchip.v0 +# queuepush stderr "------- end of stderr ---------" +# queuesave stderr $options:0/log.ckchip.v0 Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip_s3.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip_s3.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip_s3.pt (revision 42443) @@ -0,0 +1,117 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_CKCHIP_S3_INIT)) + book init ckchip_s3.book + $RAWFIX_CKCHIP_S3_INIT = 1 +end + +# select nights in state ckchip_s3.new +# output: ckchip_s3.book +task nights.ckchip_s3.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/ckchip_s3.load.stdout.log + stderr $LOGDIR/ckchip_s3.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state ckchip_s3.new + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip_s3.book + book delpage ckchip_s3.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.ckchip_s3.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages ckchip_s3.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in ckchip_s3.book (pantaskState == INIT) + book getpage ckchip_s3.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + stdout $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 + stderr $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 + + book setword ckchip_s3.book $pageName pantaskState RUN + book getword ckchip_s3.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command $mypath/check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v1 + end + + # SUCCESS + task.exit 0 + book setword ckchip_s3.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 ckchip_s3.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword ckchip_s3.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 ckchip_s3.fail + end + + task.exit crash + showcommand crash + book setword ckchip_s3.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 ckchip_s3.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword ckchip_s3.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 ckchip_s3.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip.pt (revision 42443) @@ -0,0 +1,115 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_INIT)) + book init fixchip.book + $RAWFIX_FIXCHIP_INIT = 1 +end + +# select nights in state fixchip.new +# output: fixchip.book +task nights.fixchip.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip.load.stdout.log + stderr $LOGDIR/fixchip.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book + book delpage fixchip.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip.book (pantaskState == INIT) + book getpage fixchip.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip.book $pageName pantaskState RUN + book getword fixchip.book $pageName dateobs -var DATEOBS + option $pageName + + stdout $pageName/log.fixchip.v0 + stderr $pageName/log.fixchip.v0 + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + book setword fixchip.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip.fail + end + + task.exit crash + showcommand crash + book setword fixchip.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s2.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s2.pt (revision 42443) @@ -0,0 +1,119 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_S2_INIT)) + book init fixchip_s2.book + $RAWFIX_FIXCHIP_S2_INIT = 1 +end + +# select nights in state fixchip_s2.new +# output: fixchip_s2.book +task nights.fixchip_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip_s2.load.stdout.log + stderr $LOGDIR/fixchip_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s2.book + book delpage fixchip_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip_s2.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip_s2.book (pantaskState == INIT) + book getpage fixchip_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip_s2.book $pageName pantaskState RUN + book getword fixchip_s2.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 + stderr $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 + + command $mypath/fix_chip_locations_stage2.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword fixchip_s2.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip_s2.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip_s2.fail + end + + task.exit crash + showcommand crash + book setword fixchip_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip_s2.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s3.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s3.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s3.pt (revision 42443) @@ -0,0 +1,119 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_S3_INIT)) + book init fixchip_s3.book + $RAWFIX_FIXCHIP_S3_INIT = 1 +end + +# select nights in state fixchip_s3.new +# output: fixchip_s3.book +task nights.fixchip_s3.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip_s3.load.stdout.log + stderr $LOGDIR/fixchip_s3.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip_s3.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s3.book + book delpage fixchip_s3.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip_s3.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip_s3.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip_s3.book (pantaskState == INIT) + book getpage fixchip_s3.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip_s3.book $pageName pantaskState RUN + book getword fixchip_s3.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 + stderr $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 + + command $mypath/fix_chip_locations_stage3.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword fixchip_s3.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip_s3.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip_s3.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip_s3.fail + end + + task.exit crash + showcommand crash + book setword fixchip_s3.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip_s3.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip_s3.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip_s3.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk.pt (revision 42443) @@ -0,0 +1,118 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD + +if (not($?RAWFIX_MD5CHK_INIT)) + book init md5chk.book + $RAWFIX_MD5CHK_INIT = 1 +end + +# select nights in state md5chk.new +# output: md5chk.book +task nights.md5chk.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5chk.load.stdout.log + stderr $LOGDIR/md5chk.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state md5chk.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk.book + book delpage md5chk.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.md5chk.run + host anyhost + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5chk.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5chk.book (pantaskState == INIT) + book getpage md5chk.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5chk.book $pageName pantaskState RUN + book getword md5chk.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5chk.v0 + stderr $mypath/$DATEOBS/log.md5chk.v0 + + command $mypath/check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword md5chk.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 md5chk.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5chk.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 md5chk.fail + end + + task.exit crash + showcommand crash + book setword md5chk.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 md5chk.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5chk.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 md5chk.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk_s2.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk_s2.pt (revision 42443) @@ -0,0 +1,118 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD + +if (not($?RAWFIX_MD5CHK_S2_INIT)) + book init md5chk_s2.book + $RAWFIX_MD5CHK_S2_INIT = 1 +end + +# select nights in state md5chk_s2.new +# output: md5chk_s2.book +task nights.md5chk_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5chk_s2.load.stdout.log + stderr $LOGDIR/md5chk_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state md5chk_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk_s2.book + book delpage md5chk_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.md5chk_s2.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5chk_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5chk_s2.book (pantaskState == INIT) + book getpage md5chk_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5chk_s2.book $pageName pantaskState RUN + book getword md5chk_s2.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5chk_s2.v0 + stderr $mypath/$DATEOBS/log.md5chk_s2.v0 + + command $mypath/check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword md5chk_s2.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 md5chk_s2.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5chk_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 md5chk_s2.fail + end + + task.exit crash + showcommand crash + book setword md5chk_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 md5chk_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5chk_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 md5chk_s2.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum.pt (revision 42443) @@ -0,0 +1,287 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum calculations +## use the top-level rawfix.pt to load + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_INIT)) + book init md5sum.book + book init md5host.book + $RAWFIX_MD5SUM_INIT = 1 +end + +# select nights which are ready for md5sum calcs +# output: md5sum.book +task nights.md5sum.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5sum.load.stdout.log + stderr $LOGDIR/md5sum.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state md5sum.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book + book delpage md5sum.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: md5host.book +task nights.md5sum.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host local + npending 1 + # note: the tasks which return their results via stdout + # seem to fail if run remotely. + + stdout $LOGDIR/md5sum.loadhosts.stdout.log + stderr $LOGDIR/md5sum.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5sum.book (pantaskState == INIT) + book getpage md5sum.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5sum.book $pageName pantaskState RUN + book getword md5sum.book $pageName dateobs -var DATEOBS + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.md5sum.runhosts.v0" $mypath $DATEOBS + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + command $mypath/get_hosts_md5s.sh $mypath $DATEOBS v0 + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queueprint stdout + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book + book setword md5sum.book $options:0 NHOST {$Nhosts - 1} + book setword md5sum.book $options:0 NDONE 0 + book list + book listbook md5sum.book + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum.book $options:0 pantasksQuality TIMEOUT + end +end + +# runs on the hosts identified by get_hosts_md5s.sh +# output: md5host.book +task nights.md5sum.runhosts + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5host.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in md5host.book (pantaskState == INIT) + book getpage md5host.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5host.book $pageName pantaskState RUN + book getword md5host.book $pageName dateobs -var DATEOBS + book getword md5host.book $pageName md5host -var MD5HOST + + option $pageName $DATEOBS + # save DATEOBS so we can find the md5sum.book entry + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5sum.runhosts.v0 + stderr $mypath/$DATEOBS/log.md5sum.runhosts.v0 + + # XXX TEST: $USE_HOST = ipp060 + $USE_HOST = $MD5HOST + host -required $USE_HOST + ## check if this host needs to be launched + if (not($?host:$USE_HOST)) + echo "host not defined, creating : $USE_HOST" + $host:$USE_HOST = 0 + end + if (not($host:$USE_HOST)) + control host add $USE_HOST + $host:$USE_HOST = 1 + end + + ## even when testing, use the original hostname here: + $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt + + # need to make sure this will work on the ipp b-nodes: + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input --stage v0 + end + + task.exit 0 + nights.md5sum.exit OK + end + + # exit values other than 0: + task.exit default + nights.md5sum.exit FAIL + showcommand failure + end + + task.exit crash + nights.md5sum.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.md5sum.exit TIMEOUT + showcommand timeout + end +end + +# output: md5host.book +task nights.md5sum.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5sum.clean.stdout.log + stderr $LOGDIR/md5sum.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in md5sum.book (pantaskState == DONE) + book getpage md5sum.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage md5sum.book $pageName + + # book setword md5sum.book $pageName pantaskState RUN + book getword md5sum.book $pageName dateobs -var DATEOBS + book getword md5sum.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS md5sum.done + else + command update.rawfix.sh $DATEOBS md5sum.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword md5sum.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.md5sum.exit + if ($0 != 2) + echo "USAGE: nights.md5sum.exit (quality)" + break + end + + book setword md5host.book $options:0 pantaskState DONE + book getword md5sum.book $options:1 NHOST -var Nhosts + book getword md5sum.book $options:1 NDONE -var Ndone + $Ndone ++ + book setword md5sum.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword md5sum.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/log.md5sum.runhosts.v0" $mypath $options:1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword md5sum.book $options:1 pantaskState STOP + end + book delpage md5host.book $options:0 +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum_s2.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum_s2.pt (revision 42443) @@ -0,0 +1,283 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum_s2 calculations +## use the top-level rawfix.pt to load + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_S2_INIT)) + book init md5sum_s2.book + book init md5host_s2.book + $RAWFIX_MD5SUM_S2_INIT = 1 +end + +# select nights which are ready for md5sum_s2 calcs +# output: md5sum_s2.book +task nights.md5sum_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5sum_s2.load.stdout.log + stderr $LOGDIR/md5sum_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state md5sum_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum_s2.book + book delpage md5sum_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host local + npending 1 + + stdout $LOGDIR/md5sum_s2.loadhosts.stdout.log + stderr $LOGDIR/md5sum_s2.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5sum_s2.book (pantaskState == INIT) + book getpage md5sum_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5sum_s2.book $pageName pantaskState RUN + book getword md5sum_s2.book $pageName dateobs -var DATEOBS + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $DATEOBS + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + command $mypath/get_hosts_md5s.sh $mypath $DATEOBS v0 + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host_s2.book + book setword md5sum_s2.book $options:0 NHOST {$Nhosts - 1} + book setword md5sum_s2.book $options:0 NDONE 0 + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum_s2.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum_s2.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.runhosts + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5host_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in md5host_s2.book (pantaskState == INIT) + book getpage md5host_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5host_s2.book $pageName pantaskState RUN + book getword md5host_s2.book $pageName dateobs -var DATEOBS + book getword md5host_s2.book $pageName md5host -var MD5HOST + + option $pageName $DATEOBS + # save DATEOBS so we can find the md5sum_s2.book entry + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 + stderr $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 + + # XXX TEST: $USE_HOST = ipp060 + + $USE_HOST = $MD5HOST + host -required $USE_HOST + ## check if this host needs to be launched + if (not($?host:$USE_HOST)) + echo "host not defined, creating : $USE_HOST" + $host:$USE_HOST = 0 + end + if (not($host:$USE_HOST)) + control host add $USE_HOST + $host:$USE_HOST = 1 + end + + ## even when testing, use the original hostname here: + $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt + + # need to make sure this will work on the ipp b-nodes: + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command $mypath/get_md5s_instances_stage2.pl --hostname $USE_HOST --input $input --stage v0 + end + + # dump stdout / stderr to a file on failure? + task.exit 0 + nights.md5sum_s2.exit OK + end + + # exit values other than 0: + task.exit default + nights.md5sum_s2.exit FAIL + showcommand failure + end + + task.exit crash + nights.md5sum_s2.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.md5sum_s2.exit TIMEOUT + showcommand timeout + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5sum_s2.clean.stdout.log + stderr $LOGDIR/md5sum_s2.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in md5sum_s2.book (pantaskState == DONE) + book getpage md5sum_s2.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage md5sum_s2.book $pageName + + # book setword md5sum_s2.book $pageName pantaskState RUN + book getword md5sum_s2.book $pageName dateobs -var DATEOBS + book getword md5sum_s2.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS md5sum_s2.done + else + command update.rawfix.sh $DATEOBS md5sum_s2.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword md5sum_s2.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum_s2.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum_s2.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.md5sum_s2.exit + if ($0 != 2) + echo "USAGE: nights.md5sum_s2.exit (quality)" + break + end + + book setword md5host_s2.book $options:0 pantaskState DONE + book getword md5sum_s2.book $options:1 NHOST -var Nhosts + book getword md5sum_s2.book $options:1 NDONE -var Ndone + $Ndone ++ + book setword md5sum_s2.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword md5sum_s2.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $options:1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword md5sum_s2.book $options:1 pantaskState STOP + end + book delpage md5host_s2.book $options:0 +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.pt (revision 42443) @@ -0,0 +1,63 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup + +$TPOLL = 5 +$TEXEC = 5 +$RPOLL = 5 +$REXEC = 5 +$VERBOSE = 1 +$NETWORK = 1 + +# use this to find the files in this local directory +# $mypath = /home/panstarrs/eugene/rawfix.20230221 + +# NOTE: use a directory located on a b-node so the results can +# be seen by both ATRC and ITC machines +$mypath = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?LOGDIR)) + $LOGDIR = $mypath/pantasks_logs + mkdir $LOGDIR +end + +# these are listed in stage order, but the +# order is this file is not determinative, only the +# change of state from STAGE1.new -> STAGE1.done -> STAGE2.new +input rawfix.advance.pt +input rawfix.ckchip.pt +input rawfix.fixchip.pt +input rawfix.md5sum.pt +input rawfix.md5chk.pt +input rawfix.fixchip_s2.pt +input rawfix.md5sum_s2.pt +input rawfix.md5chk_s2.pt +input rawfix.fixchip_s3.pt +input rawfix.ckchip_s3.pt + +# no need to run rsync if we use the b-node working directory: +# input rawfix.rsync_s1.pt +# input rawfix.rsync_s2.pt + +######################### UTILITIES ################################# + +macro showcommand -c "print out the current command (used within tasks)" + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + date -var exitdate + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + echo "job exit date: $exitdate" + else + echo "command: $command" + end + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s1.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s1.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s1.pt (revision 42443) @@ -0,0 +1,126 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- rsync -auv $SRCDIR/ $TGTDIR/ + +$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221 +$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?RAWFIX_RSYNC_S1_INIT)) + book init rsync_s1.book + $RAWFIX_RSYNC_S1_INIT = 1 +end + +# select nights in state rsync_s1.new +# output: rsync_s1.book +task nights.rsync_s1.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/rsync_s1.load.stdout.log + stderr $LOGDIR/rsync_s1.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state rsync_s1.new + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s1.book + book delpage rsync_s1.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +###########################################################3333333 + +task nights.rsync_s1.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/rsync_s1.run.stdout.log + stderr $LOGDIR/rsync_s1.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages rsync_s1.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in rsync_s1.book (pantaskState == INIT) + book getpage rsync_s1.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # echo "got a valid entry: $pageName" + # book listpage rsync_s1.book $pageName + + book setword rsync_s1.book $pageName pantaskState RUN + book getword rsync_s1.book $pageName dateobs -var DATEOBS + option $pageName + + # send output to a file in the DATEOBS directory + stdout $DATEOBS/rsync_s1.log + stderr $DATEOBS/rsync_s1.log + + command rsync -auv $SRCDIR/$DATEOBS/ $TGTDIR/$DATEOBS/ + end + + ## do not delete the page from the book if done here + ## deletion has to happen after the .load + task.exit 0 + echo "--- stdout : nights.rsync_s1.run ---" + queueprint stdout + exec update.rawfix.sh $options:0 rsync_s1.done + book setword rsync_s1.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword rsync_s1.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 rsync_s1.fail + end + + task.exit crash + showcommand crash + book setword rsync_s1.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 rsync_s1.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword rsync_s1.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 rsync_s1.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s2.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s2.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s2.pt (revision 42443) @@ -0,0 +1,124 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- rsync -auv $SRCDIR/ $TGTDIR/ + +$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221 +$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?RAWFIX_RSYNC_S2_INIT)) + book init rsync_s2.book + $RAWFIX_RSYNC_S2_INIT = 1 +end + +# select nights in state rsync_s2.new +# output: rsync_s2.book +task nights.rsync_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/rsync_s2.load.stdout.log + stderr $LOGDIR/rsync_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state rsync_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s2.book + book delpage rsync_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +###########################################################3333333 + +task nights.rsync_s2.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/rsync_s2.run.stdout.log + stderr $LOGDIR/rsync_s2.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages rsync_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in rsync_s2.book (pantaskState == INIT) + book getpage rsync_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # echo "got a valid entry: $pageName" + # book listpage rsync_s2.book $pageName + + book setword rsync_s2.book $pageName pantaskState RUN + book getword rsync_s2.book $pageName dateobs -var DATEOBS + option $pageName + + # send output to a file in the DATEOBS directory + stdout $DATEOBS/rsync_s2.log + stderr $DATEOBS/rsync_s2.log + + command rsync -auv $TGTDIR/$DATEOBS/ $SRCDIR/$DATEOBS/ + end + + ## do not delete the page from the book if done here + ## deletion has to happen after the .load + task.exit 0 + echo "--- stdout : nights.rsync_s2.run ---" + queueprint stdout + exec update.rawfix.sh $options:0 rsync_s2.done + book setword rsync_s2.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword rsync_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 rsync_s2.fail + end + + task.exit crash + showcommand crash + book setword rsync_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 rsync_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword rsync_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 rsync_s2.fail + end +end + Index: /trunk/tools/eam/rawfix.20230221/src.v0/select.nights.sql =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/select.nights.sql (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/select.nights.sql (revision 42443) @@ -0,0 +1,1 @@ +select count(datename) as Nexp, datename from (select exp_name, filter, date_format(dateobs,"%Y%m%d") as datename from rawExp where (dateobs >= '2010/05/01') and (dateobs < '2011/01/01')) as tmp group by datename order by Nexp desc Index: /trunk/tools/eam/rawfix.20230221/src.v0/task.test.pt =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/task.test.pt (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/task.test.pt (revision 42443) @@ -0,0 +1,59 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum calculations +## use the top-level rawfix.pt to load + +$RPOLL = 1 +$REXEC = 1 + +# use this to find the files in this local directory +# $mypath = /home/panstarrs/eugene/rawfix.20230221 + +# NOTE: use a directory located on a b-node so the results can +# be seen by both ATRC and ITC machines +$mypath = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?LOGDIR)) + $LOGDIR = $mypath/pantasks_logs + mkdir $LOGDIR +end + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_INIT)) + book init md5sum.book + book init md5host.book + $RAWFIX_MD5SUM_INIT = 1 +end + +# output: md5host.book +task nights.md5sum.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + #stdout test.loadhosts.stdout.log + #stderr test.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 1000 + + command $mypath/get_hosts_md5s.sh $mypath 20101129 v0 + end + + task.exit 0 + queuesize stdout -var Nhosts + for i 0 $Nhosts + queuepop stdout -var line:$i + end + echo "done with nights.md5sum.loadhosts, $Nhosts" + queueprint stdout + end +end + +## It looks like stdout buffer is not completely filled, and i +## it has an extra line at the start (stdout 1) Index: /trunk/tools/eam/rawfix.20230221/src.v0/triple.fix.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/triple.fix.pl (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/triple.fix.pl (revision 42443) @@ -0,0 +1,196 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($movelist); + +GetOptions( 'list=s' => \$movelist) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = '/home/panstarrs/eugene/triples.20201211/nebulous.config'; + +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + # print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine list of files which have been moved", -exitval => 2) unless + defined $movelist; + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# load the list of files from movelist + +# the movelist files generated by 'find_triples.pl' has three header lines +# followed by a list of ext_id only (no so_id) + +open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +my @movefiles = ; +close (MOVELIST); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; + +# skip the first 3 header lines +for (my $i = 3; $i < @movefiles; $i++) { + + my $movefile = $movefiles[$i]; + chomp $movefile; + + my @word = split (" ", $movefile); + + my $so_id; + my $ext_id; + if (0) { + # output from rebalance code has so_id ext_id, but not find_triples.pl + # so_id ext_id + $so_id = $word[0]; + $ext_id = $word[1]; + } else { + $so_id = 0; + $ext_id = $word[0]; + } + + # if ( ! defined ($so_id)) { + # print STDERR "ERROR : INVALID SO_ID: $movefile\n"; + # next; + # } + + if ( ! defined ($ext_id)) { + print STDERR "ERROR : MISSING EXT_ID: $movefile\n"; + next; + } + + #### LIMIT to FITS, SMF, CMF #### + # for now, only do *.fits files: + my $is_valid = 0; + if ($ext_id =~ m|fits$|) { $is_valid = 1; } + if ($ext_id =~ m|cmf$|) { $is_valid = 1; } + if ($ext_id =~ m|smf$|) { $is_valid = 1; } + + # skip files which do not match desired rules: + unless ($is_valid) { next; } + + # my $summary = "$ext_id $so_id"; + my $summary = "$ext_id"; + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + if (1) { + my $ncopies_query = "SELECT ins_id, vol_id, xattr FROM instance JOIN volume USING (vol_id) JOIN storage_object using (so_id) WHERE (ext_id = '${ext_id}')"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find an instance + print STDERR "ERROR: NO INSTANCES: $summary\n"; + next; + } + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $array[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 3) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + my $xattr = $values[2]; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + } + } + $summary = $summary . " $Nmain $Nback"; + + # SKIP : already fixed + if (($Nback == 1) && ($Nmain == 1)) { + print STDERR "SKIP 1: BACKUP AND MAIN OK: $summary\n"; + next; + } + + # SKIP : too many backups, how to cull them? + if ($Nback > 1) { + print STDERR "SKIP 2: TOO MANY BACKUPS : $summary\n"; + next; + } + + # ITC files with a single backup copies need to be culled + # this case was SKIP 3: + if (($Nback == 1) && ($Nmain > 1)) { + my $Ncull = $Nmain - 1; + print STDERR "CULL: TOO MANY MAIN BUT BACKUP OK, CULL $Ncull X: $summary\n"; + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; } + } + next; + } + + # single copy @ ITC + if (($Nmain == 1) && ($Nback == 0)) { + print STDERR "SING: MAKE BACKUP : $summary\n"; + my $status = 0; + $status = vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL SN1: neb-replicate : $summary\n"; next; } + next; + } + + # multiple copies @ ITC + if (($Nmain > 1) && ($Nback == 0)) { + my $Ncull = $Nmain - 1; + print STDERR "DUPS: MAKE BACKUP AND CULL $Ncull X : $summary\n"; + my $status = 0; + $status = vsystem("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL TR1: neb-replicate : $summary\n"; next; } + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; } + } + next; + } + + print STDERR "ERROR: NO BACKUP AND NO MAIN: $summary\n"; +} # END loop over movefiles + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src.v0/update.bnodes.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/update.bnodes.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/update.bnodes.sh (revision 42443) @@ -0,0 +1,22 @@ +#!/bin/csh + +if ($#argv != 2) then + echo "USAGE: update.bnodes.sh (push/pull) (test/commit)" + exit 2 +endif + +set options = "-auvn --exclude=trash" +if ($2 == "commit") set options = "-auv" + +if ($1 == "push") then + rsync $options ./ ippb07:rawfix.20230221/ + exit 0 +endif + +if ($1 == "pull") then + rsync $options ippb07:rawfix.20230221/ ./ + exit 0 +endif + +echo "ERROR: unknown mode $1" +exit 2 Index: /trunk/tools/eam/rawfix.20230221/src.v0/update.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src.v0/update.rawfix.sh (revision 42443) +++ /trunk/tools/eam/rawfix.20230221/src.v0/update.rawfix.sh (revision 42443) @@ -0,0 +1,21 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: update.rawfix.sh (night) (state) [quality]" + echo " EG: update.rawfix.sh 20100501 new" + exit 2 +endif + +set mynight = $1 +set state = $2 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" + +mysql $DBOPTS -e "update nights set state = '$state' where (dateobs = '$mynight')" +exit 0