Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/Makefile =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/Makefile (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/Makefile (revision 42531) @@ -0,0 +1,60 @@ + +help: + @echo "make options: install" + +BIN = ~/src/bin/rawfix +SRC = src + +PANTASKS = \ +rawfix.pt \ +rawfix.advance.pt \ +rawfix.ckchip.pt \ +rawfix.fixchip.pt \ +rawfix.md5sum.pt \ +rawfix.md5chk.pt \ +rawfix.burntool.pt + +# interfaces with the rawfix database +DB_PROG = \ +check.rawfix.sh \ +queue.rawfix.sh \ +update.rawfix.sh + +# these are in order used by the tasks +PERL_PROG = \ +advance.rawfix.pl \ +check_chip_locations.pl \ +get_hosts_fixchip.sh \ +fix_chip_locations.pl \ +get_hosts_md5s.sh \ +get_md5s_instances.pl \ +check_md5s_dateobs.pl \ +psvideostats.sh \ +ipp_apply_burntool_psvideo.pl + +# user tools, but not used by the tasks +TOOLS = \ +update.bnodes.sh \ +check_night_status.pl \ +neb_check_location.pl \ +update_chip_stats.pl \ +finish.fixchip.sh + +# not done, or not needed now +MISC = \ +triple.fix.pl \ +fix_chip_failures.pl \ +create.rawfix.sh + +PROGRAMS = $(PANTASKS) $(DB_PROG) $(PERL_PROG) $(TOOLS) + +install: $(PROGRAMS) + +$(PROGRAMS): % : $(BIN)/% + +$(BIN)/% : $(SRC)/% + @if [ ! -d $(BIN) ]; then mkdir -p $(BIN); fi + rm -f $(BIN)/$* + cp $(SRC)/$* $(BIN)/$* + + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/advance.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/advance.rawfix.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/advance.rawfix.sh (revision 42531) @@ -0,0 +1,94 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: advance.rawfix.sh all (Nmax)" + echo "USAGE: advance.rawfix.sh night (YYYYMMDD)" + exit 2 +endif + +set mode = $1 + +## include a max number of active programs +if ($mode == "all") then + if ($#argv != 2) then + echo "USAGE: advance.rawfix.sh all (Nmax)" + exit 2 + endif + + set nMax = $2 + set nActive = 0 + set nAdvance = 0 + set nights = `./check.rawfix.sh all | awk '(NR > 1){print $1}'` + foreach f ($nights) + ./advance.rawfix.sh night $f + set myResult = $status + if ($myResult == 1) continue + if ($myResult == 2) continue + if ($myResult == 3) continue + if ($myResult == 4) then + @ nActive ++ + endif + if ($myResult == 5) then + @ nActive ++ + @ nAdvance ++ + endif + if ($nActive >= $nMax) then + echo "advance $nAdvance nights, total of $nActive active nights" + exit 0 + endif + end + echo "advance $nAdvance nights, total of $nActive active nights" + exit 0 +endif + +# advance STAGE0.done to STAGE1.new +# if already in STAGEx.new, return 4 +# if setting to STAGEx.new, return 5 +# otherwise, return 3 +if ($mode == "night") then + if ($#argv != 2) then + echo "USAGE: advance.rawfix.sh night (YYYYMMDD)" + exit 2 + endif + + set night = $2 + + set mystate = `./check.rawfix.sh night $2 | awk '(NR == 2){print $2}'` + + set tgtstate = "NONE" + if ($mystate == "new") set tgtstate = "ckchip.new" + if ($mystate == "ckchip.done") set tgtstate = "fixchip.new" + if ($mystate == "fixchip.done") set tgtstate = "md5sum.new" + if ($mystate == "md5sum.done") set tgtstate = "md5chk.new" + if ($mystate == "md5chk.done") set tgtstate = "fixchip_s2.new" + if ($mystate == "fixchip_s2.done") set tgtstate = "md5sum_s2.new" + if ($mystate == "md5sum_s2.done") set tgtstate = "md5chk_s2.new" + if ($mystate == "md5chk_s2.done") set tgtstate = "fixchip_s3.new" + if ($mystate == "fixchip_s3.done") set tgtstate = "ckchip_s3.new" + if ($mystate == "ckchip_s3.done") set tgtstate = "DONE" + + if ($mystate == "ckchip.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip.new") set tgtstate = "ACTIVE" + if ($mystate == "md5sum.new") set tgtstate = "ACTIVE" + if ($mystate == "md5chk.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "md5sum_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "md5chk_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip_s3.new") set tgtstate = "ACTIVE" + if ($mystate == "ckchip_s3.new") set tgtstate = "ACTIVE" + + echo "$night : $mystate : $tgtstate" + + # do not set these two states + if ($tgtstate == "NONE") exit 3 + if ($tgtstate == "ACTIVE") exit 4 + + # only set these states + ./update.rawfix.sh $night $tgtstate + + if ($tgtstate == "DONE") exit 1 + exit 5 +endif + +echo "ERROR: unknown mode: $mode" +exit 2 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check.rawfix.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check.rawfix.sh (revision 42531) @@ -0,0 +1,115 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: check.rawfix.sh (mode) [options]" + echo "USAGE: check.rawfix.sh all" + echo "USAGE: check.rawfix.sh stats" + echo "USAGE: check.rawfix.sh night (YYYYMMDD)" + echo "USAGE: check.rawfix.sh state (state)" + echo " NOTE: state = raw, rsync, md5sum.new" + exit 2 +endif + +set mode = $1 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" +set DBOPTS = "-B -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" + +if ($mode == "all") then + if ($#argv != 1) then + echo "USAGE: check.rawfix.sh all" + exit 2 + endif + + mysql $DBOPTS -e "select dateobs, state from nights" + exit 0 +endif + +if ($mode == "stats") then + if ($#argv != 1) then + echo "USAGE: check.rawfix.sh stats" + exit 2 + endif + + mysql $DBOPTS -e "select count(dateobs) Nnights, state from nights group by state" + exit 0 +endif + +if ($mode == "night") then + if ($#argv != 2) then + echo "USAGE: check.rawfix.sh night (YYYYMMDD)" + exit 2 + endif + + set mynight = $2 + mysql $DBOPTS -e "select dateobs, state from nights where (dateobs = '$mynight')" + exit 0 +endif + +if ($mode == "state") then + if ($#argv != 2) then + echo "USAGE: check.rawfix.sh state (state)" + exit 2 + endif + + set mystate = $2 + + if ($mystate == "ckchip.new") goto good_state + if ($mystate == "ckchip.done") goto good_state + if ($mystate == "ckchip.fail") goto good_state + if ($mystate == "fixchip.new") goto good_state + if ($mystate == "fixchip.done") goto good_state + if ($mystate == "fixchip.fail") goto good_state + if ($mystate == "md5sum.new") goto good_state + if ($mystate == "md5sum.done") goto good_state + if ($mystate == "md5sum.fail") goto good_state + if ($mystate == "md5chk.new") goto good_state + if ($mystate == "md5chk.done") goto good_state + if ($mystate == "md5chk.fail") goto good_state + + # stage 2: + if ($mystate == "fixchip_s2.new") goto good_state + if ($mystate == "fixchip_s2.done") goto good_state + if ($mystate == "fixchip_s2.fail") goto good_state + if ($mystate == "md5sum_s2.new") goto good_state + if ($mystate == "md5sum_s2.done") goto good_state + if ($mystate == "md5sum_s2.fail") goto good_state + if ($mystate == "md5chk_s2.new") goto good_state + if ($mystate == "md5chk_s2.done") goto good_state + if ($mystate == "md5chk_s2.fail") goto good_state + + # stage 3: + if ($mystate == "fixchip_s3.new") goto good_state + if ($mystate == "fixchip_s3.done") goto good_state + if ($mystate == "fixchip_s3.fail") goto good_state + if ($mystate == "ckchip_s3.new") goto good_state + if ($mystate == "ckchip_s3.done") goto good_state + if ($mystate == "ckchip_s3.fail") goto good_state + +# disable the rsyncs stages: run in directory from b-node +# if ($mystate == "rsync_s1.new") goto good_state +# if ($mystate == "rsync_s1.done") goto good_state +# if ($mystate == "rsync_s1.fail") goto good_state +# if ($mystate == "rsync_s2.new") goto good_state +# if ($mystate == "rsync_s2.done") goto good_state +# if ($mystate == "rsync_s2.fail") goto good_state + + echo "ERROR: unknown mode $mystate" + exit 2 + + good_state: + set t = `date +%T..%N` + mysql $DBOPTS -e "select dateobs, curtime() as timestamp from nights where (state = '$mystate')" | awk -v T=$t '(NR == 1){print $0}(NR > 1){print $1, T}' + exit 0 +endif + +echo "ERROR: unknown mode $mode" +exit 2 + + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_chip_locations.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_chip_locations.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_chip_locations.pl (revision 42531) @@ -0,0 +1,423 @@ +#! /usr/bin/env perl + +# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases +my $DEBUG = 3; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) +# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage v0 + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($neb_dbh, $gpc_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# a global to store the number of instances per host (for each chip) +my %hostcount; +my %hostfile; + +# put output files in a directory defined by the night +mkdir "$topdir/$dateword"; + +# output files: +# instance lists by host: YYYYMMDD/HOSTNAME.inst.STAGE.txt +# chip states by chip: YYYYMMDD/CHIPNAME.uris.txt + +foreach my $chip_id (@chip_ids) { + + my $outfh; + open ($outfh, ">$topdir/$dateword/$chip_id.uris.txt"); + + # get the list of chip neb paths for one chip + my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id); + + # I think this makes a copy of the arrays, which is inefficient + # my @neb_paths = @$neb_paths_r; + # my @ds_array = @$ds_array_r; + # my @md_array = @$md_array_r; + + # reset the hostcount for this chip + %hostcount = (); + + # find the neb instances for each chip nebulous entry + + for (my $i = 0; $i < @$neb_paths_r; $i++) { + my $nebname = $neb_paths_r->[$i]; # equivalent to ${$neb_paths_r}[$i] + my $data_st = $ds_array_r->[$i]; + my $md5_sum = $md_array_r->[$i]; + + # print STDERR "nebname: $nebname, dstate: $data_st, md5: $md5_sum\n"; + + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($nebname); + + # determine the status of the instances + my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum); + + &check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Ndead); + } + + close ($outfh); + + # now save the host count for this chip + my $hostfh; + open ($hostfh, ">$topdir/$dateword/$chip_id.hosts.txt"); + + # save the host count for this chip + my @hostnames = keys %hostcount; + foreach my $myname (@hostnames) { + print $hostfh "$myname $hostcount{$myname}\n"; + } + close ($hostfh); +} + +# save the host count for this chip +my @hostnames = keys %hostfile; +foreach my $myname (@hostnames) { + close ($hostfile{$myname}); + if ($DEBUG > 4) { print "close host instance file for $myname\n"; } +} + +# run neb-replicate for SING entries? +# run neb-replicate for SBCK entries? (does this put a second copy at ATRC or ITC?) +# save the list of instances on each host +## run md5sum on the instances for each host +## only note inconsistent md5sums? (need to save the gpc1 md5sum values) +# rerun to generate a list of the host with max instances per chip? + +### when to cull entries? +## neb-replicate puts the new entry at ATRC by default +## neb-replicate --volume VOL puts the new entry at the specified location +### in order to use this, I need to choose the target volume + +exit 0; + +# die "done"; + +sub vsystem { + # print STDERR "@_\n"; + return 0; + #my $status = system ("@_"); + #return $status; +} + + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +# get the list of chip neb paths for one chip +sub get_chip_paths { + my $fulldate = $_[0]; + my $chip_id = $_[1]; + + my (@nebarray, @ds_array, @md_array); + + my $sql_chip_paths = "SELECT uri, data_state, md5sum from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'"; + if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } + + my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows chip images\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only uri selected above) + my $Nvalues = @values; + if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; } + + push (@nebarray, $values[0]); + push (@ds_array, $values[1]); + push (@md_array, $values[2]); + if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; } + } + + # return references so we can keep the three arrays distinct + return (\@nebarray, \@ds_array, \@md_array); +} + +sub get_chip_instances { + + my $nebname = $_[0]; + + # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + + # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + if ($DEBUG > 5) { print "nebname: $ext_id\n"; } + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# count the number of instances at ITC (0), ATRC (1), or other locations (>1) +# record the instances in a file for each host +# count the number of instances on each host +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + my $data_st = $_[2]; + my $md5_sum = $_[3]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + my $Ndead = 0; + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + # the number of returned values per row must match the query above (in get_chip_instances) + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if ($xattr > 1) { $Ndead ++; next; } # do not save instances on dead partitions + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + + # count the number of each host by incrementing the hash element corresponding to the hostname: + # NOTE: hostcount is global + $hostcount{$hostname} ++; + + if (not defined $hostfile{$hostname}) { + open ($hostfile{$hostname}, ">$topdir/$dateword/$hostname.inst.$stage.txt"); + ## XXX deal with open failure here + } + + my $fh = $hostfile{$hostname}; + print $fh "$instance $data_st $md5_sum\n"; + } + + return ($Nmain, $Nback, $Ndead); +} + +sub check_instance_state { + + my $outfh = $_[0]; + my $extID = $_[1]; + my $data_st = $_[2]; + my $md5_sum = $_[3]; + my $Nmain = $_[4]; + my $Nback = $_[5]; + my $Ndead = $_[6]; + + my $outinfo = "$extID $data_st $md5_sum"; + + # GOOD : already fixed + if (($Nmain == 1) && ($Nback == 1)) { + print $outfh "GOOD: $outinfo\n"; + return; + } + + # SING : single copy @ ITC, no backup + # make a backup + if (($Nmain == 1) && ($Nback == 0)) { + print $outfh "SING: $outinfo\n"; + return; + } + + # SBCK : single copy @ ATRC, no main copy + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print $outfh "SBCK: $outinfo\n"; + return; + } + + # DUPS : multiple copies @ ITC, none at ATRC + # make a backup and cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 0)) { + print $outfh "DUPS: $outinfo\n"; + return; + } + + # XBCK : multiple copies @ ATRC, but none at ITC + # make a backup at ITC, cull ATRC extras + if (($Nmain == 0) && ($Nback > 1)) { + print $outfh "XBCK: $outinfo\n"; + return; + } + + # DEAD : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print $outfh "DEAD: $outinfo\n"; + return; + } + # GONE : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print $outfh "GONE: $outinfo\n"; + return; + } + + # CUL1 : multiple ITC files, single backup + # cull only ITC + if (($Nmain > 1) && ($Nback == 1)) { + print $outfh "CUL1: $outinfo\n"; + return; + } + + # CUL2 : multiple ATRC files, single ITC + # cull only ATRC + if (($Nback > 1) && ($Nmain == 1)) { + print $outfh "CUL2: $outinfo\n"; + return; + } + + # CUL3 : excess ATRC & ITC files, cull from both? + # cull Nmain - 1 + if (($Nback > 1) && ($Nmain > 1)) { + print $outfh "CUL3: $outinfo\n"; + return; + } + + # unexpected state: + print $outfh "OOPS: $outinfo\n"; + return; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir) +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; + pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; + pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; + pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; + pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; + if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } + my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh, $gpc_dbh); +} + + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_chip_locations.v0.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_chip_locations.v0.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_chip_locations.v0.pl (revision 42531) @@ -0,0 +1,423 @@ +#! /usr/bin/env perl + +# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases + +# TESTRUN can be 1, 2, 3, 4 to get further in the tests +my $TESTRUN = 2; +my $DEBUG = 3; + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($tgthost,$tgtvol,$subdir,$movelist); + +# USAGE: check_chip_locations.pl --dateobs (night) --chip_id (chip_id) +# USAGE: check_chip_locations.pl --dateobs 2010.05.01 --chip_id XY33 + +my ($dateobs, $chip_id); +GetOptions( 'dateobs=s' => \$dateobs, 'chip_id=s' => \$chip_id) || pod2usage(2); + +my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); +my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); +my $conffile = 'nebulous.config'; + +# parse the nebulous.config file: +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; +pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; +pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; +pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; +pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + +pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; +pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; +pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; +pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; +pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + +pod2usage( -msg => "Cannot determine target date", -exitval => 2) unless defined $dateobs; +pod2usage( -msg => "Cannot determine target chip", -exitval => 2) unless defined $chip_id; + +# check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + +my ($date_year, $date_month, $date_day) = split "/",$dateobs; +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + +my $time = time(); + +my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; +if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } +my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; +if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } +my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# get the list of chip neb paths: +my @nebarray; +if (1) { + my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$date_year/$date_month/$date_day,00:00:00' and dateobs < '$date_year/$date_month/$date_day,23:59:59'"; + if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } + + my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows chip images\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only uri selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@nebarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } +} + +# find the neb instances for each chip nebulous entry + +my %hostcount; + +foreach my $nebname (@nebarray) { + + # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + if ($DEBUG > 5) { print "nebname: $ext_id\n"; } + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + if (0) { + foreach my $valref (@tmparray) { + my @values = @$valref; + # number of values should match list above (can this even fail?) + my $Nvalues = @values; + if ($Nvalues != 5) { print "ERROR: wrong number of fields?\n"; next; } + + if ($DEBUG > 2) { print "uri: $values[0], host: $values[1], avail: $values[2], xattr: $values[3], cab: $values[4]\n"; } + } + } + + # my $summary = "$ext_id $so_id"; + my $summary = "$ext_id"; + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + my $Ndead = 0; + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $tmparray[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + my $xattr = $values[3]; + my $hostname = $values[1]; + + # count the number of each host by incrementing the hash element corresponding to the hostname: + $hostcount{$hostname} ++; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + if ($xattr > 1) { $Ndead ++; } + } + + # SKIP : already fixed + if (($Nmain == 1) && ($Nback == 1)) { + print STDOUT "GOOD: $summary\n"; + next; + } + + # single copy @ ITC, no backup + # make a backup + if (($Nmain == 1) && ($Nback == 0)) { + print STDOUT "SING: $summary\n"; + my $status = vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } + next; + } + + # ITC files with a single backup copies need to be culled + # cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 1)) { + my $Ncull = $Nmain - 1; + print STDOUT "CULL: $summary\n"; + for (my $j = 0; $j < $Ncull; $j ++) { + my $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } + } + next; + } + + # multiple copies @ ITC + # make a backup and cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 0)) { + my $Ncull = $Nmain - 1; + print STDOUT "DUPS: $summary\n"; + my $status = vsystem("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } + } + next; + } + + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print STDOUT "DEAD: $summary\n"; + next; + } + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print STDOUT "GONE: $summary\n"; + next; + } + + # single copy @ ATRC + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print STDOUT "SBCK: $summary\n"; + next; + } + + # multiple copies @ ATRC, but none at ITC + # make a backup at ITC, cull ATRC extras + if (($Nmain == 0) && ($Nback > 1)) { + print STDOUT "XBCK: $summary\n"; + next; + } + + # unexpected state: + print STDOUT "OOPS: $summary\n"; + next; +} + +my @hostnames = keys %hostcount; +foreach my $myname (@hostnames) { + print "host $myname = $hostcount{$myname}\n"; +} + +die "done"; + +#my $query = $dbh->prepare( $sql_get_volume_id ); + +#$query->execute( $tgt_volume ); +#my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +#$query->finish; + + +### XX +### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; + +# load the list of files from movelist + +##XX open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +##XX my @movefiles = ; +##XX close (MOVELIST); +##XX +##XX my $otime = $time; +##XX $time = time(); +##XX my $dtime = $time - $otime; +##XX +##XX for (my $i = 0; $i < @movefiles; $i++) { +##XX +##XX my $movefile = $movefiles[$i]; +##XX chomp $movefile; +##XX +##XX # the filename should have the following form: +##XX # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: +##XX # NN : 2-hexdigit name of the subsubdir +##XX # DDDDD : instance ID (long) +##XX # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name +##XX +##XX # I want to check two things: +##XX # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile +##XX # 2) does the vol_id of the instance_id match our vol_id ? +##XX +##XX unless (-e "$tgt_subdir/$movefile") { +##XX print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; +##XX next; +##XX } +##XX +##XX # print "movefile: $movefile\n"; +##XX +##XX my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; +##XX my $summary = "$tgt_subdir/$movefile"; +##XX +##XX if ( ! defined ($ins_id)) { +##XX print "ERROR : INVALID ID: $summary\n"; +##XX next; +##XX } +##XX +##XX if ( $ins_id =~ m/\D/ ) { +##XX print "ERROR : INVALID ID: $summary\n"; +##XX next; +##XX } +##XX $summary = "$summary : $ins_id"; +##XX +##XX # get so_id for this ins_id: +##XX my $so_id = 0; +##XX if (1) { +##XX my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $so_id_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # We should find this instance +##XX print "ERROR : NO DB ENTRY: $summary\n"; +##XX next; +##XX } +##XX +##XX if ($Nrows > 1) { # We should only find one instance +##XX print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; +##XX next; +##XX } +##XX +##XX my $valref = $array[0]; +##XX my @values = @$valref; +##XX +##XX my $Nvalue = @values; +##XX +##XX if ($Nvalue != 1) { +##XX print "ERROR : INVALID DB RESPONSE: $summary\n"; +##XX next; +##XX } +##XX $so_id = $values[0]; +##XX } +##XX +##XX $summary = "$summary : $so_id"; +##XX +##XX # get user_copies for this storage_object +##XX my $Ncopies = 0; +##XX my $ext_id = ""; +##XX if (1) { +##XX my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # user.copies is not set, skip +##XX print "SKIP: NO USER.COPIES ENTRY: $summary\n"; +##XX next; +##XX } +##XX +##XX if ($Nrows > 1) { # We should only find one instance +##XX print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; +##XX next; +##XX } +##XX +##XX my $valref = $array[0]; +##XX my @values = @$valref; +##XX +##XX my $Nvalue = @values; +##XX +##XX if ($Nvalue != 2) { +##XX print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; +##XX next; +##XX } +##XX $ext_id = $values[0]; +##XX $Ncopies = $values[1]; +##XX } +##XX +##XX $summary = "$summary : $ext_id"; +##XX +##XX if ($Ncopies < 2) { +##XX print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; +##XX next; +##XX } +##XX +##XX # check for backup instances +##XX if (1) { +##XX my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # We should find this instance +##XX print "REPL: NO BACKUP COPY: $summary\n"; +##XX next; +##XX } +##XX +##XX ## my $valref = $array[0]; +##XX ## my @values = @$valref; +##XX ## +##XX ## my $Nvalue = @values; +##XX ## +##XX ## if ($Nvalue != 1) { +##XX ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; +##XX ## next; +##XX ## } +##XX } +##XX +##XX } # END loop over movefiles + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + +sub vsystem { + # print STDERR "@_\n"; + return 0; + #my $status = system ("@_"); + #return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_md5s_dateobs.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_md5s_dateobs.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_md5s_dateobs.pl (revision 42531) @@ -0,0 +1,127 @@ +#! /usr/bin/env perl + +my $DEBUG = 1; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_md5s_dateobs.pl --dateobs (night) --topdir (topdir) + +my ($dateword, $stage, $topdir) = &parse_cmdopts; + +my @instlist = glob ("$topdir/$dateword/*.inst.$stage.txt"); +my @md5slist = glob ("$topdir/$dateword/*.md5s.txt"); + +my $Nfail = 0; + +foreach my $instfile (@instlist) { + + # find the matching md5sum file for this host + + my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|; + # print STDERR "host: $host\n"; + + my $md5match = "NONE"; + foreach my $md5sfile (@md5slist) { + + my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|; + # print STDERR "md5host: $md5shost\n"; + + if ($host ne $md5shost) { next; } + + $md5match = $md5sfile; + last; + } + + if ($md5match eq "NONE") { + print STDOUT "missing md5sum file for $instfile\n"; + $Nfail ++; + next; + } + + ## read in the md5sums + open (FILE, $md5match); + my @md5data = ; + close (FILE); + + ## read in the list of instances + open (FILE, $instfile); + my @instdata = ; + close (FILE); + + # if these do not match, complain and move on + my $N1 = @md5data; + my $N2 = @instdata; + if ($N1 != $N2) { + print STDOUT "inconsistent number of lines in $md5match : $N1 vs $N2\n"; + $Nfail ++; + next; + } + + # check for mismatched MD5sums + foreach my $line (@md5data) { + + chomp $line; + my @words = split (/\s+/,$line); + + # we expect lines of the form: INST STATE MDtru MDobs + if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; } + + my $inst = $words[0]; + my $state = $words[1]; + my $md5tru = $words[2]; + my $md5obs = $words[3]; + + if ($md5tru ne $md5obs) { + print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; + $Nfail ++; + } else { + if ($DEBUG > 2) { + print STDOUT "$inst $state $md5tru = $md5obs\n"; + } + } + } +} + +if ($Nfail) { + print STDOUT "ERROR: $Nfail errors\n"; + exit 1; +} else { + print STDOUT "SUCCESS: $Nfail errors\n"; + exit 0; +} + +sub parse_cmdopts { + + my ($dateobs, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($dateword, $stage, $topdir); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_night_status.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_night_status.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/check_night_status.pl (revision 42531) @@ -0,0 +1,131 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_night_status.pl --dateobs (night) --chiplist (file) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$dateword/$chip_id.uris.txt"; + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist); +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/create.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/create.rawfix.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/create.rawfix.sh (revision 42531) @@ -0,0 +1,14 @@ +#!/bin/csh + +# make the rawfix database to track rawfix operations +set DBNAME = "rawfix" +set DBUSER = "dvo" + +mysql -h localhost -u root < qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_failures.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) +# USAGE: fix_chip_failures.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage v0 + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; + + +my (@instance, @data_state, @md5tru, @md5obs) = &load_failures; + +sub load_failures { + + my (@instance, @data_state, @md5tru, @md5obs); + + # given a night, load the failures from log.md5chk.v0 + open (INLOG, "$topdir/$dateword/log.md5chk.$stage"); + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + unless ($line =~ /^FAIL:/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 5) { die "error in log output $line\n"; } + + push (@instance, $words[1]); + push (@data_state, $words[2]); + push (@md5tru, $words[3]); + push (@md5obs, $words[4]); + } + close (CHIPS); + + return (@instance, @data_state, @md5tru, @md5obs); +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir) +} + +# identify the + +sub neb_reverse { + + my (@neb_path) = split /\//, $file; + my $neb_key = $neb_path[-1]; + # Remove ins_id + $neb_key =~ s/^\d+\.//; + # Convert colons to slashes. + $neb_key =~ s%:%/%g; + $neb_key =~ s%.fz$%%; + print "$neb_key\n"; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations.pl (revision 42531) @@ -0,0 +1,253 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# choose ITC machines on which to place backup copies: +my @volarray = &get_neb_volumes; +my $volnum = 0; + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; + open (FILE, $urifile); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # one copy at ITC: neb-replicate will create a copy at ATRC + if ($state eq "SING:") { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL SING: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nsing ++; + next; + } + + # one copy at ATRC: neb-replicate will create a copy at ITC iff we supply the target volume + if ($state eq "SBCK:") { + my $tgtvol = $volarray[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL SBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nsbck ++; + next; + } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + $volnum ++; +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_new_instance { + + my $ext_id = $_[0]; + + ## the first instance should be the last one created: + my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $uri = $values[0]; + my $host = $values[1]; + my $xattr = $values[2]; + + return ($uri, $host, $xattr); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my @volarray; + + my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@volarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @volarray; +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage2.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage2.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage2.pl (revision 42531) @@ -0,0 +1,253 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (v0,v1,etc) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# choose ITC machines on which to place backup copies: +my @volarray = &get_neb_volumes; +my $volnum = 0; + +my $Ngood = 0; +my $Ndead = 0; +my $Ngone = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; + open (FILE, $urifile); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these were done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } # one copy at ITC: should not happen at stage 2 + if ($state eq "SBCK:") { $Nsbck++; next; } # one copy at ATRC: should not happen at stage 2 + + # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies + if ($state eq "DUPS:") { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Ndups++; + next; + } + + # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume + if ($state eq "XBCK:") { + my $tgtvol = $volarray[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$topdir/$dateword/$newhost.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + $Nxbck++; + next; + } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + $volnum ++; +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_new_instance { + + my $ext_id = $_[0]; + + ## the first instance should be the last one created: + my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $uri = $values[0]; + my $host = $values[1]; + my $xattr = $values[2]; + + return ($uri, $host, $xattr); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my @volarray; + + my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@volarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @volarray; +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage3.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage3.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/fix_chip_locations_stage3.pl (revision 42531) @@ -0,0 +1,286 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations_stage3.pl --dateobs (night) --chiplist (file) + +my ($fulldate, $dateword, $chiplist, $topdir) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); +my %volarray = &get_neb_volumes; + +# choose ITC machines on which to place backup copies: +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$topdir/$dateword/$chip_id.uris.txt"; + open (FILE, $urifile); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + my $datast = $words[2]; + my $md5sum = $words[3]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these were replicated in stage 2, so actually match one of the CUL1 or CUL2 entries + # if ($state eq "DUPS:") { $Ndups++; next; } + # if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + # if ($state eq "CUL1:") { $Ncul1++; next; } + # if ($state eq "CUL2:") { $Ncul2++; next; } + # if ($state eq "CUL3:") { $Ncul3++; next; } + + # one copy at ITC: neb-replicate will create a copy at ATRC + if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:") || ($state eq "DUPS:") || ($state eq "XBCK:")) { + + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($ext_id); + + my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id); + + if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n"; next; } + if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; } + + my $NcullTotal = @$cull_list_r; + foreach my $instance (@$cull_list_r) { + my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|; + unless (defined $volarray{$volname}) { die "ERROR: invalid volume name $volname\n"; } + # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n"; + my $status = &vsystem ("neb-cull --volume $volname $ext_id"); + if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; } + } + # print STDERR "Ncull Total $NcullTotal\n"; + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +sub get_chip_instances { + + my $ext_id = $_[0]; + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# select 1 instance with xattr = 0, 1 with xattr = 1. +# the rest should be culled. + +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $main_keep = ""; + my $back_keep = ""; + my @cull_list = (); + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + # the number of returned values per row must match the query above (in get_chip_instances) + my $Nvalue = @values; + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; } + if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; } + + # if we make it here, we already have an instance at this location + push (@cull_list, $instance); + } + return ($main_keep, $back_keep, \@cull_list); +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +# +sub parse_cmdopts { + + my ($dateobs, $chiplist, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $topdir); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my %volarray; + + my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + $volarray{$values[0]} = 1; + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return %volarray; +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_hosts_md5s.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_hosts_md5s.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_hosts_md5s.sh (revision 42531) @@ -0,0 +1,24 @@ +#!/bin/csh + +if ($#argv != 3) then + echo "USAGE: get_hosts_md5s.sh (topdir) (dateobs) (stage)" + exit 2 +endif + +set topdir = $1 +set dateobs = $2 +set stage = $3 + +if (! -d $topdir) then + echo "ERROR: missing top-level directory $topdir" + exit 3 +endif + +if (! -d $topdir/$dateobs) then + echo "ERROR: missing dateobs directory $topdir/$dateobs" + exit 3 +endif + +# report both the dateobs and the hostnames +ls $topdir/$dateobs/*.inst.$stage.txt | awk -v dateobs=$dateobs -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $NF}' | sed s/.inst.$stage.txt// +exit 0 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_dateobs.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_dateobs.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_dateobs.pl (revision 42531) @@ -0,0 +1,50 @@ +#! /usr/bin/env perl + +die "XXX not done yet"; + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_dateobs.pl --dateobs (night) + +my ($fulldate, $dateword) = &parse_cmdopts; + +# first, ensure the dateobs directory is + + + +sub parse_cmdopts { + + my ($dateobs); + GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_instances.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_instances.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_instances.pl (revision 42531) @@ -0,0 +1,107 @@ +#!/usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc) + +my ($myhost, $input, $stage) = &parse_cmdopts; + +# check that we are running this script on 'myhost' +my $truhost = `hostname -s`; chomp $truhost; +if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } + +# read the list of instances: +my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); + +my $output = $input; +$output =~ s/inst.$stage/md5s/; + +if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } +if (-e $output) { + # check that all instances are in the output file? + print "$output exists, skipping\n"; + exit 0; +} + +open (OUTFILE, ">$output"); + +# foreach my $instance (@$ins_list_r) { +for (my $i = 0; $i < @$ins_list_r; $i ++) { + + my $instance = $ins_list_r->[$i]; + my $data_state = $dst_list_r->[$i]; + my $md5sum_tru = $md5_list_r->[$i]; + + my $filename = $instance; + $filename =~ s|file://||; + + # check for the existence of the file first: + unless (-e $filename) { + print OUTFILE "$instance $data_state $md5sum_tru XXX\n"; + next; + } + + my $line; + if (0) { + $line = "fakevalue $filename"; # XXX TEST: disable calculation: + } else { + $line = `md5sum $filename`; + } + + my ($md5sum, $rawfile) = split (" ", $line); + print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n"; +} +close OUTFILE; +exit 0; + +# read the list of chip_ids: +sub load_instance_list { + my $infile = $_[0]; + + my (@ins_list, @dst_list, @md5_list); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 3) { die "error in input $infile : $line\n"; } + push (@ins_list, $words[0]); + push (@dst_list, $words[1]); + push (@md5_list, $words[2]); + } + close (FILE); + + return (\@ins_list, \@dst_list, \@md5_list); +} + +sub parse_cmdopts { + + my ($myhost, $input, $stage); + GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + + return ($myhost, $input, $stage); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_instances_stage2.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_instances_stage2.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/get_md5s_instances_stage2.pl (revision 42531) @@ -0,0 +1,140 @@ +#!/usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (v0,v1,etc) + +my ($myhost, $input, $stage) = &parse_cmdopts; + +# check that we are running this script on 'myhost' +my $truhost = `hostname -s`; chomp $truhost; +if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } + +# read the list of instances: +my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); + +my $output = $input; +$output =~ s/inst.$stage/md5s/; + +if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } + +# read in both instance +my $out_hash_r; +if (-e $output) { + $out_hash_r = &load_output_list ($output); +} + +open (OUTFILE, ">>$output"); + +# foreach my $instance (@$ins_list_r) { +for (my $i = 0; $i < @$ins_list_r; $i ++) { + + my $instance = $ins_list_r->[$i]; + my $data_state = $dst_list_r->[$i]; + my $md5sum_tru = $md5_list_r->[$i]; + + # if the instance is already in the output file, skip it + if (defined $out_hash_r) { + my $name = $out_hash_r->{$instance}; + if (defined $name) { + print STDERR "skipping $instance (already in output)\n"; + next; + } + } + + my $filename = $instance; + $filename =~ s|file://||; + + # check for the existence of the file first: + unless (-e $filename) { + print OUTFILE "$instance $data_state $md5sum_tru XXX\n"; + next; + } + + my $line; + if (0) { + $line = "fakevalue $filename"; # XXX TEST: disable calculation: + } else { + $line = `md5sum $filename`; + } + + my ($md5sum, $rawfile) = split (" ", $line); + print OUTFILE "$instance $data_state $md5sum_tru $md5sum\n"; +} +close OUTFILE; +exit 0; + +# read the instance info +sub load_instance_list { + my $infile = $_[0]; + + my (@ins_list, @dst_list, @md5_list); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 3) { die "error in input $infile : $line\n"; } + push (@ins_list, $words[0]); + push (@dst_list, $words[1]); + push (@md5_list, $words[2]); + } + close (FILE); + + return (\@ins_list, \@dst_list, \@md5_list); +} + +# read the instance list in the output file +sub load_output_list { + my $infile = $_[0]; + + my (%out_hash); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 4) { die "error in input $infile : $line\n"; } + $out_hash{$words[0]} = 1; + } + close (FILE); + + return (\%out_hash); +} + +sub parse_cmdopts { + + my ($myhost, $input, $stage); + GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + pod2usage( -msg => "Define the stage (v0, v1, etc) with --stage", -exitval => 2) unless defined $stage; + + return ($myhost, $input, $stage); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/gpc1.chips.txt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/gpc1.chips.txt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/gpc1.chips.txt (revision 42531) @@ -0,0 +1,60 @@ +XY01 +XY02 +XY03 +XY04 +XY05 +XY06 +XY10 +XY11 +XY12 +XY13 +XY14 +XY15 +XY16 +XY17 +XY20 +XY21 +XY22 +XY23 +XY24 +XY25 +XY26 +XY27 +XY30 +XY31 +XY32 +XY33 +XY34 +XY35 +XY36 +XY37 +XY40 +XY41 +XY42 +XY43 +XY44 +XY45 +XY46 +XY47 +XY50 +XY51 +XY52 +XY53 +XY54 +XY55 +XY56 +XY57 +XY60 +XY61 +XY62 +XY63 +XY64 +XY65 +XY66 +XY67 +XY71 +XY72 +XY73 +XY74 +XY75 +XY76 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/neb_check_location.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/neb_check_location.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/neb_check_location.pl (revision 42531) @@ -0,0 +1,244 @@ +#! /usr/bin/env perl + +# TESTRUN can be 1, 2, 3, 4 to get further in the tests +my $TESTRUN = 2; + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($tgthost,$tgtvol,$subdir,$movelist); + +GetOptions( + 'tgthost=s' => \$tgthost, + 'tgtvol=s' => \$tgtvol, + 'subdir=s' => \$subdir, + 'list=s' => \$movelist) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = 'nebulous.config'; + +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine target machine and volume", -exitval => 2) unless + defined $tgthost && defined $tgtvol; + +pod2usage( -msg => "Cannot determine subdir or list of files which have been moved", -exitval => 2) unless + defined $movelist && defined $subdir; + +my $host_check = `hostname`; chomp($host_check); + +if ($host_check ne $tgthost) { die "Required to run on $tgthost, not on $host_check\n"; } + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# example: +# tgthost : ipp130 +# tgtvol : 0 + +# Get the vol_id of the (local) destination. + +my $tgt_volume = "${tgthost}.${tgtvol}"; +my $tgt_subdir = "/export/${tgthost}.${tgtvol}/nebulous/${subdir}"; + +### XX my $sql_get_volume_id = "SELECT vol_id, name, host, path FROM volume WHERE name = ?"; +### XX my $sql_update_instance = "UPDATE instance SET vol_id = ?, uri = ? WHERE ins_id = ?"; +### XX +### XX my $query = $dbh->prepare( $sql_get_volume_id ); +### XX +### XX $query->execute( $tgt_volume ); +### XX my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +### XX $query->finish; +### XX +### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; + +# load the list of files from movelist + +open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +my @movefiles = ; +close (MOVELIST); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; + +for (my $i = 0; $i < @movefiles; $i++) { + + my $movefile = $movefiles[$i]; + chomp $movefile; + + # the filename should have the following form: + # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: + # NN : 2-hexdigit name of the subsubdir + # DDDDD : instance ID (long) + # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name + + # I want to check two things: + # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile + # 2) does the vol_id of the instance_id match our vol_id ? + + unless (-e "$tgt_subdir/$movefile") { + print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; + next; + } + + # print "movefile: $movefile\n"; + + my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; + my $summary = "$tgt_subdir/$movefile"; + + if ( ! defined ($ins_id)) { + print "ERROR : INVALID ID: $summary\n"; + next; + } + + if ( $ins_id =~ m/\D/ ) { + print "ERROR : INVALID ID: $summary\n"; + next; + } + $summary = "$summary : $ins_id"; + + # get so_id for this ins_id: + my $so_id = 0; + if (1) { + my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; + + my $rowref = $dbh->selectall_arrayref( $so_id_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "ERROR : NO DB ENTRY: $summary\n"; + next; + } + + if ($Nrows > 1) { # We should only find one instance + print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; + next; + } + + my $valref = $array[0]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 1) { + print "ERROR : INVALID DB RESPONSE: $summary\n"; + next; + } + $so_id = $values[0]; + } + + $summary = "$summary : $so_id"; + + # get user_copies for this storage_object + my $Ncopies = 0; + my $ext_id = ""; + if (1) { + my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # user.copies is not set, skip + print "SKIP: NO USER.COPIES ENTRY: $summary\n"; + next; + } + + if ($Nrows > 1) { # We should only find one instance + print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; + next; + } + + my $valref = $array[0]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 2) { + print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + $ext_id = $values[0]; + $Ncopies = $values[1]; + } + + $summary = "$summary : $ext_id"; + + if ($Ncopies < 2) { + print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; + next; + } + + # check for backup instances + if (1) { + my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "REPL: NO BACKUP COPY: $summary\n"; + next; + } + + ## my $valref = $array[0]; + ## my @values = @$valref; + ## + ## my $Nvalue = @values; + ## + ## if ($Nvalue != 1) { + ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + ## next; + ## } + } + +} # END loop over movefiles + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/queue.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/queue.rawfix.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/queue.rawfix.sh (revision 42531) @@ -0,0 +1,41 @@ +#!/bin/csh + +if ($#argv != 2) then + echo "USAGE: queue.rawfix.sh (list) (filename)" + echo "USAGE: queue.rawfix.sh (night) (YYYYMMDD)" + exit 2 +endif + +set mode = $1 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +if ($mode == "list") then + set filename = $2 + + # validate the list: could be DATE STATE or just DATE (prcol 1)? + + # just a big list of dates? + foreach night (`cat $filename`) + queue.rawfix.sh night $night + end + exit 0 +endif + +if ($mode == "night") then + ## XXX validate the format (YYYYMMDD)? + + XXXXX: + + set mynight = $2 + + mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')" + exit 0 +endif + +echo "ERROR: unknown mode $mode" +exit 2 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.advance.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.advance.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.advance.pt (revision 42531) @@ -0,0 +1,40 @@ + +$MAX_ACTIVE_NIGHTS = 2 + +## regularly run the command advance.rawfits.sh +task nights.advance + periods -poll 10 + periods -exec 10 + periods -timeout 30 + host local + + stdout $LOGDIR/advance.log + stderr $LOGDIR/advance.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # + command advance.rawfix.sh all $MAX_ACTIVE_NIGHTS + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip.pt (revision 42531) @@ -0,0 +1,124 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_CKCHIP_INIT)) + book init ckchip.book + $RAWFIX_CKCHIP_INIT = 1 +end + +# select nights in state ckchip.new +# output: ckchip.book +task nights.ckchip.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/ckchip.load.stdout.log + stderr $LOGDIR/ckchip.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state ckchip.new + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book + book delpage ckchip.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.ckchip.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages ckchip.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in ckchip.book (pantaskState == INIT) + book getpage ckchip.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword ckchip.book $pageName pantaskState RUN + book getword ckchip.book $pageName dateobs -var DATEOBS + option $pageName + + # this is the first place in the proces that this location is known + mkdir $mypath/$DATEOBS + stdout $mypath/$DATEOBS/log.ckchip.runhosts.v0 + stderr $mypath/$DATEOBS/log.ckchip.runhosts.v0 + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command $mypath/check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 + end + + # mark the entry done, will be deleted above in .load + task.exit 0 + book setword ckchip.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 ckchip.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword ckchip.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 ckchip.fail + end + + task.exit crash + showcommand crash + book setword ckchip.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 ckchip.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword ckchip.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 ckchip.fail + end +end + +# option for writing stdout / stderr to a file +# queuepush stdout "------- end of stdout ---------" +# queuesave stdout $options:0/log.ckchip.v0 +# queuepush stderr "------- end of stderr ---------" +# queuesave stderr $options:0/log.ckchip.v0 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip_s3.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip_s3.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.ckchip_s3.pt (revision 42531) @@ -0,0 +1,117 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_CKCHIP_S3_INIT)) + book init ckchip_s3.book + $RAWFIX_CKCHIP_S3_INIT = 1 +end + +# select nights in state ckchip_s3.new +# output: ckchip_s3.book +task nights.ckchip_s3.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/ckchip_s3.load.stdout.log + stderr $LOGDIR/ckchip_s3.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state ckchip_s3.new + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip_s3.book + book delpage ckchip_s3.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.ckchip_s3.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages ckchip_s3.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in ckchip_s3.book (pantaskState == INIT) + book getpage ckchip_s3.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + stdout $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 + stderr $mypath/$DATEOBS/log.ckchip_s3.runhosts.v0 + + book setword ckchip_s3.book $pageName pantaskState RUN + book getword ckchip_s3.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command $mypath/check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v1 + end + + # SUCCESS + task.exit 0 + book setword ckchip_s3.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 ckchip_s3.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword ckchip_s3.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 ckchip_s3.fail + end + + task.exit crash + showcommand crash + book setword ckchip_s3.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 ckchip_s3.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword ckchip_s3.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 ckchip_s3.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip.pt (revision 42531) @@ -0,0 +1,115 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_INIT)) + book init fixchip.book + $RAWFIX_FIXCHIP_INIT = 1 +end + +# select nights in state fixchip.new +# output: fixchip.book +task nights.fixchip.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip.load.stdout.log + stderr $LOGDIR/fixchip.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book + book delpage fixchip.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip.book (pantaskState == INIT) + book getpage fixchip.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip.book $pageName pantaskState RUN + book getword fixchip.book $pageName dateobs -var DATEOBS + option $pageName + + stdout $pageName/log.fixchip.v0 + stderr $pageName/log.fixchip.v0 + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + book setword fixchip.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip.fail + end + + task.exit crash + showcommand crash + book setword fixchip.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s2.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s2.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s2.pt (revision 42531) @@ -0,0 +1,119 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_S2_INIT)) + book init fixchip_s2.book + $RAWFIX_FIXCHIP_S2_INIT = 1 +end + +# select nights in state fixchip_s2.new +# output: fixchip_s2.book +task nights.fixchip_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip_s2.load.stdout.log + stderr $LOGDIR/fixchip_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s2.book + book delpage fixchip_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip_s2.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip_s2.book (pantaskState == INIT) + book getpage fixchip_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip_s2.book $pageName pantaskState RUN + book getword fixchip_s2.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 + stderr $mypath/$DATEOBS/log.fixchip_s2.runhosts.v0 + + command $mypath/fix_chip_locations_stage2.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword fixchip_s2.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip_s2.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip_s2.fail + end + + task.exit crash + showcommand crash + book setword fixchip_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip_s2.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s3.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s3.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.fixchip_s3.pt (revision 42531) @@ -0,0 +1,119 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_chip_locations.pl --dateobs YYYY/MM/DD --chiplist file.txt +# -- fix_chip_locations.pl --dateobs (YYYY/MM/DD) + +if (not($?RAWFIX_FIXCHIP_S3_INIT)) + book init fixchip_s3.book + $RAWFIX_FIXCHIP_S3_INIT = 1 +end + +# select nights in state fixchip_s3.new +# output: fixchip_s3.book +task nights.fixchip_s3.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/fixchip_s3.load.stdout.log + stderr $LOGDIR/fixchip_s3.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state fixchip_s3.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip_s3.book + book delpage fixchip_s3.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.fixchip_s3.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip_s3.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixchip_s3.book (pantaskState == INIT) + book getpage fixchip_s3.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixchip_s3.book $pageName pantaskState RUN + book getword fixchip_s3.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 + stderr $mypath/$DATEOBS/log.fixchip_s3.runhosts.v0 + + command $mypath/fix_chip_locations_stage3.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword fixchip_s3.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 fixchip_s3.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip_s3.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 fixchip_s3.fail + end + + task.exit crash + showcommand crash + book setword fixchip_s3.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 fixchip_s3.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip_s3.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 fixchip_s3.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk.pt (revision 42531) @@ -0,0 +1,118 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD + +if (not($?RAWFIX_MD5CHK_INIT)) + book init md5chk.book + $RAWFIX_MD5CHK_INIT = 1 +end + +# select nights in state md5chk.new +# output: md5chk.book +task nights.md5chk.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5chk.load.stdout.log + stderr $LOGDIR/md5chk.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state md5chk.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk.book + book delpage md5chk.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.md5chk.run + host anyhost + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5chk.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5chk.book (pantaskState == INIT) + book getpage md5chk.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5chk.book $pageName pantaskState RUN + book getword md5chk.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5chk.v0 + stderr $mypath/$DATEOBS/log.md5chk.v0 + + command $mypath/check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword md5chk.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 md5chk.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5chk.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 md5chk.fail + end + + task.exit crash + showcommand crash + book setword md5chk.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 md5chk.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5chk.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 md5chk.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk_s2.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk_s2.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5chk_s2.pt (revision 42531) @@ -0,0 +1,118 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD + +if (not($?RAWFIX_MD5CHK_S2_INIT)) + book init md5chk_s2.book + $RAWFIX_MD5CHK_S2_INIT = 1 +end + +# select nights in state md5chk_s2.new +# output: md5chk_s2.book +task nights.md5chk_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5chk_s2.load.stdout.log + stderr $LOGDIR/md5chk_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state md5chk_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk_s2.book + book delpage md5chk_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.md5chk_s2.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5chk_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5chk_s2.book (pantaskState == INIT) + book getpage md5chk_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5chk_s2.book $pageName pantaskState RUN + book getword md5chk_s2.book $pageName dateobs -var DATEOBS + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5chk_s2.v0 + stderr $mypath/$DATEOBS/log.md5chk_s2.v0 + + command $mypath/check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage v0 + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword md5chk_s2.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 md5chk_s2.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5chk_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 md5chk_s2.fail + end + + task.exit crash + showcommand crash + book setword md5chk_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 md5chk_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5chk_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 md5chk_s2.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum.pt (revision 42531) @@ -0,0 +1,287 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum calculations +## use the top-level rawfix.pt to load + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_INIT)) + book init md5sum.book + book init md5host.book + $RAWFIX_MD5SUM_INIT = 1 +end + +# select nights which are ready for md5sum calcs +# output: md5sum.book +task nights.md5sum.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5sum.load.stdout.log + stderr $LOGDIR/md5sum.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state md5sum.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book + book delpage md5sum.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: md5host.book +task nights.md5sum.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host local + npending 1 + # note: the tasks which return their results via stdout + # seem to fail if run remotely. + + stdout $LOGDIR/md5sum.loadhosts.stdout.log + stderr $LOGDIR/md5sum.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5sum.book (pantaskState == INIT) + book getpage md5sum.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5sum.book $pageName pantaskState RUN + book getword md5sum.book $pageName dateobs -var DATEOBS + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.md5sum.runhosts.v0" $mypath $DATEOBS + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + command $mypath/get_hosts_md5s.sh $mypath $DATEOBS v0 + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queueprint stdout + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book + book setword md5sum.book $options:0 NHOST {$Nhosts - 1} + book setword md5sum.book $options:0 NDONE 0 + book list + book listbook md5sum.book + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum.book $options:0 pantasksQuality TIMEOUT + end +end + +# runs on the hosts identified by get_hosts_md5s.sh +# output: md5host.book +task nights.md5sum.runhosts + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5host.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in md5host.book (pantaskState == INIT) + book getpage md5host.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5host.book $pageName pantaskState RUN + book getword md5host.book $pageName dateobs -var DATEOBS + book getword md5host.book $pageName md5host -var MD5HOST + + option $pageName $DATEOBS + # save DATEOBS so we can find the md5sum.book entry + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5sum.runhosts.v0 + stderr $mypath/$DATEOBS/log.md5sum.runhosts.v0 + + # XXX TEST: $USE_HOST = ipp060 + $USE_HOST = $MD5HOST + host -required $USE_HOST + ## check if this host needs to be launched + if (not($?host:$USE_HOST)) + echo "host not defined, creating : $USE_HOST" + $host:$USE_HOST = 0 + end + if (not($host:$USE_HOST)) + control host add $USE_HOST + $host:$USE_HOST = 1 + end + + ## even when testing, use the original hostname here: + $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt + + # need to make sure this will work on the ipp b-nodes: + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input --stage v0 + end + + task.exit 0 + nights.md5sum.exit OK + end + + # exit values other than 0: + task.exit default + nights.md5sum.exit FAIL + showcommand failure + end + + task.exit crash + nights.md5sum.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.md5sum.exit TIMEOUT + showcommand timeout + end +end + +# output: md5host.book +task nights.md5sum.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5sum.clean.stdout.log + stderr $LOGDIR/md5sum.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in md5sum.book (pantaskState == DONE) + book getpage md5sum.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage md5sum.book $pageName + + # book setword md5sum.book $pageName pantaskState RUN + book getword md5sum.book $pageName dateobs -var DATEOBS + book getword md5sum.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS md5sum.done + else + command update.rawfix.sh $DATEOBS md5sum.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword md5sum.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.md5sum.exit + if ($0 != 2) + echo "USAGE: nights.md5sum.exit (quality)" + break + end + + book setword md5host.book $options:0 pantaskState DONE + book getword md5sum.book $options:1 NHOST -var Nhosts + book getword md5sum.book $options:1 NDONE -var Ndone + $Ndone ++ + book setword md5sum.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword md5sum.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/log.md5sum.runhosts.v0" $mypath $options:1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword md5sum.book $options:1 pantaskState STOP + end + book delpage md5host.book $options:0 +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum_s2.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum_s2.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.md5sum_s2.pt (revision 42531) @@ -0,0 +1,283 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum_s2 calculations +## use the top-level rawfix.pt to load + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_S2_INIT)) + book init md5sum_s2.book + book init md5host_s2.book + $RAWFIX_MD5SUM_S2_INIT = 1 +end + +# select nights which are ready for md5sum_s2 calcs +# output: md5sum_s2.book +task nights.md5sum_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5sum_s2.load.stdout.log + stderr $LOGDIR/md5sum_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state md5sum_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum_s2.book + book delpage md5sum_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host local + npending 1 + + stdout $LOGDIR/md5sum_s2.loadhosts.stdout.log + stderr $LOGDIR/md5sum_s2.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5sum_s2.book (pantaskState == INIT) + book getpage md5sum_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5sum_s2.book $pageName pantaskState RUN + book getword md5sum_s2.book $pageName dateobs -var DATEOBS + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $DATEOBS + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + command $mypath/get_hosts_md5s.sh $mypath $DATEOBS v0 + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host_s2.book + book setword md5sum_s2.book $options:0 NHOST {$Nhosts - 1} + book setword md5sum_s2.book $options:0 NDONE 0 + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum_s2.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum_s2.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.runhosts + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5host_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new entries in md5host_s2.book (pantaskState == INIT) + book getpage md5host_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5host_s2.book $pageName pantaskState RUN + book getword md5host_s2.book $pageName dateobs -var DATEOBS + book getword md5host_s2.book $pageName md5host -var MD5HOST + + option $pageName $DATEOBS + # save DATEOBS so we can find the md5sum_s2.book entry + + # does the output go here? + stdout $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 + stderr $mypath/$DATEOBS/log.md5sum_s2.runhosts.v0 + + # XXX TEST: $USE_HOST = ipp060 + + $USE_HOST = $MD5HOST + host -required $USE_HOST + ## check if this host needs to be launched + if (not($?host:$USE_HOST)) + echo "host not defined, creating : $USE_HOST" + $host:$USE_HOST = 0 + end + if (not($host:$USE_HOST)) + control host add $USE_HOST + $host:$USE_HOST = 1 + end + + ## even when testing, use the original hostname here: + $input = $mypath/$DATEOBS/$MD5HOST.inst.v0.txt + + # need to make sure this will work on the ipp b-nodes: + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command $mypath/get_md5s_instances_stage2.pl --hostname $USE_HOST --input $input --stage v0 + end + + # dump stdout / stderr to a file on failure? + task.exit 0 + nights.md5sum_s2.exit OK + end + + # exit values other than 0: + task.exit default + nights.md5sum_s2.exit FAIL + showcommand failure + end + + task.exit crash + nights.md5sum_s2.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.md5sum_s2.exit TIMEOUT + showcommand timeout + end +end + +# output: md5host_s2.book +task nights.md5sum_s2.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/md5sum_s2.clean.stdout.log + stderr $LOGDIR/md5sum_s2.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5sum_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in md5sum_s2.book (pantaskState == DONE) + book getpage md5sum_s2.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage md5sum_s2.book $pageName + + # book setword md5sum_s2.book $pageName pantaskState RUN + book getword md5sum_s2.book $pageName dateobs -var DATEOBS + book getword md5sum_s2.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS md5sum_s2.done + else + command update.rawfix.sh $DATEOBS md5sum_s2.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword md5sum_s2.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5sum_s2.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword md5sum_s2.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5sum_s2.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.md5sum_s2.exit + if ($0 != 2) + echo "USAGE: nights.md5sum_s2.exit (quality)" + break + end + + book setword md5host_s2.book $options:0 pantaskState DONE + book getword md5sum_s2.book $options:1 NHOST -var Nhosts + book getword md5sum_s2.book $options:1 NDONE -var Ndone + $Ndone ++ + book setword md5sum_s2.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword md5sum_s2.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/log.md5sum_s2.runhosts.v0" $mypath $options:1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword md5sum_s2.book $options:1 pantaskState STOP + end + book delpage md5host_s2.book $options:0 +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.pt (revision 42531) @@ -0,0 +1,63 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup + +$TPOLL = 5 +$TEXEC = 5 +$RPOLL = 5 +$REXEC = 5 +$VERBOSE = 1 +$NETWORK = 1 + +# use this to find the files in this local directory +# $mypath = /home/panstarrs/eugene/rawfix.20230221 + +# NOTE: use a directory located on a b-node so the results can +# be seen by both ATRC and ITC machines +$mypath = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?LOGDIR)) + $LOGDIR = $mypath/pantasks_logs + mkdir $LOGDIR +end + +# these are listed in stage order, but the +# order is this file is not determinative, only the +# change of state from STAGE1.new -> STAGE1.done -> STAGE2.new +input rawfix.advance.pt +input rawfix.ckchip.pt +input rawfix.fixchip.pt +input rawfix.md5sum.pt +input rawfix.md5chk.pt +input rawfix.fixchip_s2.pt +input rawfix.md5sum_s2.pt +input rawfix.md5chk_s2.pt +input rawfix.fixchip_s3.pt +input rawfix.ckchip_s3.pt + +# no need to run rsync if we use the b-node working directory: +# input rawfix.rsync_s1.pt +# input rawfix.rsync_s2.pt + +######################### UTILITIES ################################# + +macro showcommand -c "print out the current command (used within tasks)" + local n + + if ($VERBOSE) + $command = $taskarg:0 + for n 1 $taskarg:n + $command = $command $taskarg:$n + end + date -var exitdate + if ($0 == 2) + echo "" + echo "$1 for: $command" + echo "job exit status: $JOB_STATUS" + echo "job host: $JOB_HOSTNAME" + echo "job dtime: $JOB_DTIME" + echo "job exit date: $exitdate" + else + echo "command: $command" + end + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s1.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s1.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s1.pt (revision 42531) @@ -0,0 +1,126 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- rsync -auv $SRCDIR/ $TGTDIR/ + +$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221 +$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?RAWFIX_RSYNC_S1_INIT)) + book init rsync_s1.book + $RAWFIX_RSYNC_S1_INIT = 1 +end + +# select nights in state rsync_s1.new +# output: rsync_s1.book +task nights.rsync_s1.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/rsync_s1.load.stdout.log + stderr $LOGDIR/rsync_s1.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state rsync_s1.new + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s1.book + book delpage rsync_s1.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +###########################################################3333333 + +task nights.rsync_s1.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/rsync_s1.run.stdout.log + stderr $LOGDIR/rsync_s1.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages rsync_s1.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in rsync_s1.book (pantaskState == INIT) + book getpage rsync_s1.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # echo "got a valid entry: $pageName" + # book listpage rsync_s1.book $pageName + + book setword rsync_s1.book $pageName pantaskState RUN + book getword rsync_s1.book $pageName dateobs -var DATEOBS + option $pageName + + # send output to a file in the DATEOBS directory + stdout $DATEOBS/rsync_s1.log + stderr $DATEOBS/rsync_s1.log + + command rsync -auv $SRCDIR/$DATEOBS/ $TGTDIR/$DATEOBS/ + end + + ## do not delete the page from the book if done here + ## deletion has to happen after the .load + task.exit 0 + echo "--- stdout : nights.rsync_s1.run ---" + queueprint stdout + exec update.rawfix.sh $options:0 rsync_s1.done + book setword rsync_s1.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword rsync_s1.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 rsync_s1.fail + end + + task.exit crash + showcommand crash + book setword rsync_s1.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 rsync_s1.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword rsync_s1.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 rsync_s1.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s2.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s2.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/rawfix.rsync_s2.pt (revision 42531) @@ -0,0 +1,124 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- rsync -auv $SRCDIR/ $TGTDIR/ + +$SRCDIR = /data/ipp060.0/eugene/rawfix.20230221 +$TGTDIR = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?RAWFIX_RSYNC_S2_INIT)) + book init rsync_s2.book + $RAWFIX_RSYNC_S2_INIT = 1 +end + +# select nights in state rsync_s2.new +# output: rsync_s2.book +task nights.rsync_s2.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/rsync_s2.load.stdout.log + stderr $LOGDIR/rsync_s2.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state rsync_s2.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout rsync_s2.book + book delpage rsync_s2.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +###########################################################3333333 + +task nights.rsync_s2.run + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/rsync_s2.run.stdout.log + stderr $LOGDIR/rsync_s2.run.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages rsync_s2.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in rsync_s2.book (pantaskState == INIT) + book getpage rsync_s2.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # echo "got a valid entry: $pageName" + # book listpage rsync_s2.book $pageName + + book setword rsync_s2.book $pageName pantaskState RUN + book getword rsync_s2.book $pageName dateobs -var DATEOBS + option $pageName + + # send output to a file in the DATEOBS directory + stdout $DATEOBS/rsync_s2.log + stderr $DATEOBS/rsync_s2.log + + command rsync -auv $TGTDIR/$DATEOBS/ $SRCDIR/$DATEOBS/ + end + + ## do not delete the page from the book if done here + ## deletion has to happen after the .load + task.exit 0 + echo "--- stdout : nights.rsync_s2.run ---" + queueprint stdout + exec update.rawfix.sh $options:0 rsync_s2.done + book setword rsync_s2.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword rsync_s2.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 rsync_s2.fail + end + + task.exit crash + showcommand crash + book setword rsync_s2.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 rsync_s2.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword rsync_s2.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 rsync_s2.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/select.nights.sql =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/select.nights.sql (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/select.nights.sql (revision 42531) @@ -0,0 +1,1 @@ +select count(datename) as Nexp, datename from (select exp_name, filter, date_format(dateobs,"%Y%m%d") as datename from rawExp where (dateobs >= '2010/05/01') and (dateobs < '2011/01/01')) as tmp group by datename order by Nexp desc Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/task.test.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/task.test.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/task.test.pt (revision 42531) @@ -0,0 +1,59 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum calculations +## use the top-level rawfix.pt to load + +$RPOLL = 1 +$REXEC = 1 + +# use this to find the files in this local directory +# $mypath = /home/panstarrs/eugene/rawfix.20230221 + +# NOTE: use a directory located on a b-node so the results can +# be seen by both ATRC and ITC machines +$mypath = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?LOGDIR)) + $LOGDIR = $mypath/pantasks_logs + mkdir $LOGDIR +end + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_INIT)) + book init md5sum.book + book init md5host.book + $RAWFIX_MD5SUM_INIT = 1 +end + +# output: md5host.book +task nights.md5sum.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + #stdout test.loadhosts.stdout.log + #stderr test.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 1000 + + command $mypath/get_hosts_md5s.sh $mypath 20101129 v0 + end + + task.exit 0 + queuesize stdout -var Nhosts + for i 0 $Nhosts + queuepop stdout -var line:$i + end + echo "done with nights.md5sum.loadhosts, $Nhosts" + queueprint stdout + end +end + +## It looks like stdout buffer is not completely filled, and i +## it has an extra line at the start (stdout 1) Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/triple.fix.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/triple.fix.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/triple.fix.pl (revision 42531) @@ -0,0 +1,196 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($movelist); + +GetOptions( 'list=s' => \$movelist) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = '/home/panstarrs/eugene/triples.20201211/nebulous.config'; + +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + # print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine list of files which have been moved", -exitval => 2) unless + defined $movelist; + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# load the list of files from movelist + +# the movelist files generated by 'find_triples.pl' has three header lines +# followed by a list of ext_id only (no so_id) + +open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +my @movefiles = ; +close (MOVELIST); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; + +# skip the first 3 header lines +for (my $i = 3; $i < @movefiles; $i++) { + + my $movefile = $movefiles[$i]; + chomp $movefile; + + my @word = split (" ", $movefile); + + my $so_id; + my $ext_id; + if (0) { + # output from rebalance code has so_id ext_id, but not find_triples.pl + # so_id ext_id + $so_id = $word[0]; + $ext_id = $word[1]; + } else { + $so_id = 0; + $ext_id = $word[0]; + } + + # if ( ! defined ($so_id)) { + # print STDERR "ERROR : INVALID SO_ID: $movefile\n"; + # next; + # } + + if ( ! defined ($ext_id)) { + print STDERR "ERROR : MISSING EXT_ID: $movefile\n"; + next; + } + + #### LIMIT to FITS, SMF, CMF #### + # for now, only do *.fits files: + my $is_valid = 0; + if ($ext_id =~ m|fits$|) { $is_valid = 1; } + if ($ext_id =~ m|cmf$|) { $is_valid = 1; } + if ($ext_id =~ m|smf$|) { $is_valid = 1; } + + # skip files which do not match desired rules: + unless ($is_valid) { next; } + + # my $summary = "$ext_id $so_id"; + my $summary = "$ext_id"; + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + if (1) { + my $ncopies_query = "SELECT ins_id, vol_id, xattr FROM instance JOIN volume USING (vol_id) JOIN storage_object using (so_id) WHERE (ext_id = '${ext_id}')"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find an instance + print STDERR "ERROR: NO INSTANCES: $summary\n"; + next; + } + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $array[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 3) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + my $xattr = $values[2]; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + } + } + $summary = $summary . " $Nmain $Nback"; + + # SKIP : already fixed + if (($Nback == 1) && ($Nmain == 1)) { + print STDERR "SKIP 1: BACKUP AND MAIN OK: $summary\n"; + next; + } + + # SKIP : too many backups, how to cull them? + if ($Nback > 1) { + print STDERR "SKIP 2: TOO MANY BACKUPS : $summary\n"; + next; + } + + # ITC files with a single backup copies need to be culled + # this case was SKIP 3: + if (($Nback == 1) && ($Nmain > 1)) { + my $Ncull = $Nmain - 1; + print STDERR "CULL: TOO MANY MAIN BUT BACKUP OK, CULL $Ncull X: $summary\n"; + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; } + } + next; + } + + # single copy @ ITC + if (($Nmain == 1) && ($Nback == 0)) { + print STDERR "SING: MAKE BACKUP : $summary\n"; + my $status = 0; + $status = vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL SN1: neb-replicate : $summary\n"; next; } + next; + } + + # multiple copies @ ITC + if (($Nmain > 1) && ($Nback == 0)) { + my $Ncull = $Nmain - 1; + print STDERR "DUPS: MAKE BACKUP AND CULL $Ncull X : $summary\n"; + my $status = 0; + $status = vsystem("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL TR1: neb-replicate : $summary\n"; next; } + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; } + } + next; + } + + print STDERR "ERROR: NO BACKUP AND NO MAIN: $summary\n"; +} # END loop over movefiles + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/update.bnodes.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/update.bnodes.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/update.bnodes.sh (revision 42531) @@ -0,0 +1,22 @@ +#!/bin/csh + +if ($#argv != 2) then + echo "USAGE: update.bnodes.sh (push/pull) (test/commit)" + exit 2 +endif + +set options = "-auvn --exclude=trash" +if ($2 == "commit") set options = "-auv" + +if ($1 == "push") then + rsync $options ./ ippb07:rawfix.20230221/ + exit 0 +endif + +if ($1 == "pull") then + rsync $options ippb07:rawfix.20230221/ ./ + exit 0 +endif + +echo "ERROR: unknown mode $1" +exit 2 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/update.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/update.rawfix.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src.v0/update.rawfix.sh (revision 42531) @@ -0,0 +1,21 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: update.rawfix.sh (night) (state) [quality]" + echo " EG: update.rawfix.sh 20100501 new" + exit 2 +endif + +set mynight = $1 +set state = $2 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" + +mysql $DBOPTS -e "update nights set state = '$state' where (dateobs = '$mynight')" +exit 0 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/advance.rawfix.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/advance.rawfix.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/advance.rawfix.pl (revision 42531) @@ -0,0 +1,269 @@ +#! /usr/bin/env perl + +my $DEBUG = 2; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: advance.rawfix.pl --mode (all/night) --dateobs (night) --topdir (dirname) --Nmax Nmax +# topdir used only to find the nebulous config + +my ($mode, $topdir, $fulldate, $dateword, $Nmax) = &parse_cmdopts; +my ($raw_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +if ($DEBUG >= 4) { + print "mode : $mode \n"; + print "topdir : $topdir \n"; + if (defined $fulldate) { print "fulldate : $fulldate\n"; } + if (defined $dateword) { print "dateword : $dateword\n"; } + if (defined $Nmax ) { print "Nmax : $Nmax \n"; } +} + +if ($mode eq "all") { + &advance_all ($Nmax); + exit 0; +} + +if ($mode eq "night") { + &advance_night ($dateword); + exit 0; +} + +die "invalid mode $mode\n"; + +sub advance_all { + + my $Nmax = $_[0]; + + # we allow $Nmax to be currently running + + # first count the number of nights already in state 'STAGE.new' + # XXX: use a list of valid .new states? + my $sql_get_new = "SELECT dateobs from nights where state like '%.new'"; + my $newref = $raw_dbh->selectall_arrayref( $sql_get_new ); + my @newarray = @$newref; + my $Nnew = @newarray; + + print "$Nnew night(s) already running\n"; + + if ($Nnew >= $Nmax) { return; } + + # select and advance nights already running + my $sql_get_run = "SELECT dateobs from nights where state like '%.done'"; + my $runref = $raw_dbh->selectall_arrayref( $sql_get_run ); + my @runarray = @$runref; + my $Nrun = @runarray; + + print "$Nrun night(s) finished a stage\n"; + + my $i = 0; + my $Ntotal = $Nnew; + while (($i < $Nrun) and ($Ntotal < $Nmax)) { + my $valref = $runarray[$i]; + my @values = @$valref; + my $dateobs = $values[0]; + + my $newstate = advance_night ($dateobs); + if ($newstate eq "NEW") { $Ntotal ++; } + $i ++; + # print "$dateobs : $i $Ntotal : $Nrun $Nmax\n"; + } + if ($Ntotal >= $Nmax) { return; } + + # select and advance nights not yet running +# $sql_get_run = "SELECT dateobs from nights where state = 'new'"; + + $sql_get_run = "SELECT dateobs FROM (SELECT dateobs, rand() AS myrand FROM nights WHERE (state = 'new')) AS tmp ORDER BY myrand"; + + $runref = $raw_dbh->selectall_arrayref( $sql_get_run ); + @runarray = @$runref; + $Nrun = @runarray; + + print "$Nrun night(s) ready to start\n"; + + $i = 0; + while (($i < $Nrun) and ($Ntotal < $Nmax)) { + my $valref = $runarray[$i]; + my @values = @$valref; + my $dateobs = $values[0]; + + my $newstate = advance_night ($dateobs); + if ($newstate eq "NEW") { $Ntotal ++; } + $i ++; + } + return; +} + +sub advance_night { + + my $dateobs = $_[0]; + + ## get the current state of this night: + my $fields = ""; + $fields .= " Nchip,"; + $fields .= " Ngood,"; + $fields .= " Ndead,"; + $fields .= " Ngone,"; + $fields .= " Nsing,"; + $fields .= " Ndups,"; + $fields .= " Nsbck,"; + $fields .= " Nxbck,"; + $fields .= " Ncul1,"; + $fields .= " Ncul2,"; + $fields .= " Ncul3,"; + $fields .= " Nfail"; + + my $sql_get_night = "SELECT dateobs, state, iter, $fields from nights where dateobs = '$dateobs'"; + my $rowref = $raw_dbh->selectall_arrayref( $sql_get_night ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + if ($Nrows != 1) { die "invalid response from database for night $dateobs\n"; } + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $datenew = $values[0]; + my $state = $values[1]; + my $iter = $values[2]; + + if ($datenew != $dateobs) { die "inconsistent result from rawfix database for $dateobs (wrong dateobs?)\n"; } + + my $Nchip = $values[3]; + my $Ngood = $values[4]; + my $Ndead = $values[5]; + my $Ngone = $values[6]; + my $Nsing = $values[7]; + my $Ndups = $values[8]; + my $Nsbck = $values[9]; + my $Nxbck = $values[10]; + my $Ncul1 = $values[11]; + my $Ncul2 = $values[12]; + my $Ncul3 = $values[13]; + my $Nfail = $values[14]; + + # in state of 'new', we do not care what the values of the stat fields are. + # they may be NULL, so do not yet try to do math on them here + if ($state eq "new") { &update_night_state ($dateobs, "ckchip.new", 0); return "NEW"; } + + my $Ndone = $Ngood + $Ndead + $Ngone + $Nfail; + my $Nwait = $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3; + + if ($Ndone + $Nwait != $Nchip) { die "inconsistent result from rawfix database for $dateobs\n"; } + + # check for completion here first (otherwise, advance fixchip.done below) + if (($state eq "fixchip.done") and ($iter > 0) and ($Nwait == 0)) { &update_night_state ($dateobs, "DONE"); return "DONE"; } + if ($state eq "ckchip.done") { &update_night_state ($dateobs, "fixchip.new"); return "NEW"; } + if ($state eq "fixchip.done") { &update_night_state ($dateobs, "md5sum.new"); return "NEW"; } + if ($state eq "md5sum.done") { &update_night_state ($dateobs, "md5chk.new"); return "NEW"; } + if ($state eq "md5chk.done") { $iter ++; &update_night_state ($dateobs, "ckchip.new", $iter); return "NEW"; } + if ($state eq "md5fix.done") { $iter ++; &update_night_state ($dateobs, "ckchip.new", $iter); return "NEW"; } + + return "NONE"; +} + +sub update_night_state { + my $dateobs = $_[0]; + my $newstate = $_[1]; + my $newiter = $_[2]; + + # update the stats in the database + my $sql_update_state; + if (defined $newiter) { + $sql_update_state = "UPDATE nights set state = '$newstate', iter = $newiter WHERE dateobs = $dateobs"; + if ($DEBUG >= 2) { print "set $dateobs to $newstate, iter = $newiter\n"; } + } else { + $sql_update_state = "UPDATE nights set state = '$newstate' WHERE dateobs = $dateobs"; + if ($DEBUG >= 2) { print "set $dateobs to $newstate\n"; } + } + + # print "SQL: $sql_update_state\n"; + # return; + + my $query = $raw_dbh->prepare($sql_update_state); + $query->execute () or die "invalid response from RAW database\n"; + return; +} + +sub parse_cmdopts { + + my ($mode, $topdir, $dateobs, $fulldate, $dateword, $Nmax); + GetOptions( 'mode=s' => \$mode, 'topdir=s' => \$topdir, 'dateobs=s' => \$dateobs, 'Nmax=i' => \$Nmax) || pod2usage(2); + + pod2usage( -msg => "Specify the mode with --mode (all or night)", -exitval => 2) unless defined $mode; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + if ($mode eq "night") { + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + $fulldate = "$date_year/$date_month/$date_day"; + $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + } + if ($mode eq "all") { + pod2usage( -msg => "Specify the maximum number of nights to advance to new with --Nmax", -exitval => 2) unless defined $Nmax; + # check that Nmax is an integer + unless (is_integer ($Nmax)) { die "Nmax must be a non-negative integer\n"; } + unless ($Nmax >= 0) { die "Nmax must be a non-negative integer\n"; } + } + if (($mode ne "all") and ($mode ne "night")) { die "invalid mode $mode\n"; } + + return ($mode, $topdir, $fulldate, $dateword, $Nmax); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /RAW_NAME /) { $rawdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_HOST /) { $rawdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_USER /) { $rawdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PASS /) { $rawdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PORT /) { $rawdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + if ($DEBUG >= 4) { print STDERR "raw dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n"; } + } + pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name; + pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host; + pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user; + pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port; + pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass; + + my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host"; + if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; } + my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($raw_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/advance.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/advance.rawfix.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/advance.rawfix.sh (revision 42531) @@ -0,0 +1,94 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: advance.rawfix.sh all (Nmax)" + echo "USAGE: advance.rawfix.sh night (YYYYMMDD)" + exit 2 +endif + +set mode = $1 + +## include a max number of active programs +if ($mode == "all") then + if ($#argv != 2) then + echo "USAGE: advance.rawfix.sh all (Nmax)" + exit 2 + endif + + set nMax = $2 + set nActive = 0 + set nAdvance = 0 + set nights = `./check.rawfix.sh all | awk '(NR > 1){print $1}'` + foreach f ($nights) + ./advance.rawfix.sh night $f + set myResult = $status + if ($myResult == 1) continue + if ($myResult == 2) continue + if ($myResult == 3) continue + if ($myResult == 4) then + @ nActive ++ + endif + if ($myResult == 5) then + @ nActive ++ + @ nAdvance ++ + endif + if ($nActive >= $nMax) then + echo "advance $nAdvance nights, total of $nActive active nights" + exit 0 + endif + end + echo "advance $nAdvance nights, total of $nActive active nights" + exit 0 +endif + +# advance STAGE0.done to STAGE1.new +# if already in STAGEx.new, return 4 +# if setting to STAGEx.new, return 5 +# otherwise, return 3 +if ($mode == "night") then + if ($#argv != 2) then + echo "USAGE: advance.rawfix.sh night (YYYYMMDD)" + exit 2 + endif + + set night = $2 + + set mystate = `./check.rawfix.sh night $2 | awk '(NR == 2){print $2}'` + + set tgtstate = "NONE" + if ($mystate == "new") set tgtstate = "ckchip.new" + if ($mystate == "ckchip.done") set tgtstate = "fixchip.new" + if ($mystate == "fixchip.done") set tgtstate = "md5sum.new" + if ($mystate == "md5sum.done") set tgtstate = "md5chk.new" + if ($mystate == "md5chk.done") set tgtstate = "fixchip_s2.new" + if ($mystate == "fixchip_s2.done") set tgtstate = "md5sum_s2.new" + if ($mystate == "md5sum_s2.done") set tgtstate = "md5chk_s2.new" + if ($mystate == "md5chk_s2.done") set tgtstate = "fixchip_s3.new" + if ($mystate == "fixchip_s3.done") set tgtstate = "ckchip_s3.new" + if ($mystate == "ckchip_s3.done") set tgtstate = "DONE" + + if ($mystate == "ckchip.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip.new") set tgtstate = "ACTIVE" + if ($mystate == "md5sum.new") set tgtstate = "ACTIVE" + if ($mystate == "md5chk.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "md5sum_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "md5chk_s2.new") set tgtstate = "ACTIVE" + if ($mystate == "fixchip_s3.new") set tgtstate = "ACTIVE" + if ($mystate == "ckchip_s3.new") set tgtstate = "ACTIVE" + + echo "$night : $mystate : $tgtstate" + + # do not set these two states + if ($tgtstate == "NONE") exit 3 + if ($tgtstate == "ACTIVE") exit 4 + + # only set these states + ./update.rawfix.sh $night $tgtstate + + if ($tgtstate == "DONE") exit 1 + exit 5 +endif + +echo "ERROR: unknown mode: $mode" +exit 2 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42531) @@ -14,11 +14,11 @@ # make the rawfix database to track rawfix operations -set DBHOST = "ipp060" +set DBHOST = "ippc190" set DBNAME = "rawfix" -set DBUSER = "dvo" -set DBPASS = "dvo" +# set DBUSER = "dvo" +# set DBPASS = "dvo" # set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" -set DBOPTS = "-B -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" +set DBOPTS = "--defaults-extra-file=~/.my.cnf.rawfix -B -h $DBHOST $DBNAME" if ($mode == "all") then @@ -28,5 +28,5 @@ endif - mysql $DBOPTS -e "select dateobs, state from nights" + mysql $DBOPTS -e "select dateobs, iter, state from nights" exit 0 endif @@ -49,5 +49,5 @@ set mynight = $2 - mysql $DBOPTS -e "select dateobs, state from nights where (dateobs = '$mynight')" + mysql $DBOPTS -E -e "select * from nights where (dateobs = '$mynight')" exit 0 endif @@ -61,15 +61,19 @@ set mystate = $2 - if ($mystate == "new") goto good_state - if ($mystate == "rsync") goto good_state - if ($mystate == "md5sum.new") goto good_state - if ($mystate == "md5sum.done") goto good_state - if ($mystate == "md5sum.fail") goto good_state - if ($mystate == "ckchip.new") goto good_state - if ($mystate == "ckchip.done") goto good_state - if ($mystate == "ckchip.fail") goto good_state - if ($mystate == "fixchip.new") goto good_state - if ($mystate == "fixchip.done") goto good_state - if ($mystate == "fixchip.fail") goto good_state + if ($mystate == "ckchip.new") goto good_state + if ($mystate == "ckchip.done") goto good_state + if ($mystate == "ckchip.fail") goto good_state + if ($mystate == "fixchip.new") goto good_state + if ($mystate == "fixchip.done") goto good_state + if ($mystate == "fixchip.fail") goto good_state + if ($mystate == "md5sum.new") goto good_state + if ($mystate == "md5sum.done") goto good_state + if ($mystate == "md5sum.fail") goto good_state + if ($mystate == "md5chk.new") goto good_state + if ($mystate == "md5chk.done") goto good_state + if ($mystate == "md5chk.fail") goto good_state + if ($mystate == "burntool.new") goto good_state + if ($mystate == "burntool.done") goto good_state + if ($mystate == "burntool.fail") goto good_state echo "ERROR: unknown mode $mystate" @@ -77,5 +81,6 @@ good_state: - mysql $DBOPTS -e "select dateobs from nights where (state = '$mystate')" + set t = `date +%T..%N` + mysql $DBOPTS -e "select dateobs, iter from nights where (state = '$mystate')" | awk -v T=$t '(NR == 1){print $0, "timestamp"}(NR > 1){print $1, $2, T}' exit 0 endif Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_chip_locations.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42531) @@ -13,9 +13,9 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) -# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt - -my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; -my ($neb_dbh, $gpc_dbh) = &parse_db_config ('nebulous.config'); +# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) +# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage 0 + +my ($fulldate, $dateword, $chiplist, $stage, $topdir, $basedir) = &parse_cmdopts; +my ($neb_dbh, $gpc_dbh, $raw_dbh) = &parse_db_config ("$topdir/nebulous.config"); # read the list of chip_ids: @@ -27,23 +27,36 @@ # put output files in a directory defined by the night -mkdir $dateword; +mkdir "$basedir/$dateword"; # output files: -# instance lists by host: YYYYMMDD/HOSTNAME.inst.txt -# chip states by chip: YYYYMMDD/CHIPNAME.uris.txt +# instance lists by host: YYYYMMDD/HOSTNAME.inst.STAGE.txt +# chip states by chip: YYYYMMDD/CHIPNAME.uris.STAGE.txt + +my $Nchip = 0; + +my $Ngood = 0; +my $Ndead = 0; +my $Ngone = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; +my $Nfail = 0; + +my ($goodchips_r, $failchips_r) = &get_good_or_fail_chiplists ($basedir, $dateword, $stage); foreach my $chip_id (@chip_ids) { - my $outfile = "$dateword/$chip_id.uris.txt"; my $outfh; - open ($outfh, ">$outfile"); + open ($outfh, ">$basedir/$dateword/$chip_id.uris.$stage.txt"); + + # print "open $basedir/$dateword/$chip_id.uris.$stage.txt\n"; # get the list of chip neb paths for one chip my ($neb_paths_r, $ds_array_r, $md_array_r) = &get_chip_paths ($fulldate, $chip_id); - - # I think this makes a copy of the arrays, which is inefficient - # my @neb_paths = @$neb_paths_r; - # my @ds_array = @$ds_array_r; - # my @md_array = @$md_array_r; # reset the hostcount for this chip @@ -65,7 +78,7 @@ # determine the status of the instances - my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum); - - &check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead); + my ($Nmain, $Nback, $Nlost) = &check_instance_xattr ($inforows, $ext_id, $data_st, $md5_sum); + + &check_instance_state ($outfh, $ext_id, $data_st, $md5_sum, $Nmain, $Nback, $Nlost); } @@ -74,10 +87,10 @@ # now save the host count for this chip my $hostfh; - open ($hostfh, ">$dateword/$chip_id.hosts.txt"); + open ($hostfh, ">$basedir/$dateword/$chip_id.hosts.txt"); # save the host count for this chip my @hostnames = keys %hostcount; foreach my $myname (@hostnames) { - print $hostfh "host $myname = $hostcount{$myname}\n"; + print $hostfh "$myname $hostcount{$myname}\n"; } close ($hostfh); @@ -91,15 +104,38 @@ } -# run neb-replicate for SING entries? -# run neb-replicate for SBCK entries? (does this put a second copy at ATRC or ITC?) -# save the list of instances on each host -## run md5sum on the instances for each host -## only note inconsistent md5sums? (need to save the gpc1 md5sum values) -# rerun to generate a list of the host with max instances per chip? - -### when to cull entries? -## neb-replicate puts the new entry at ATRC by default -## neb-replicate --volume VOL puts the new entry at the specified location -### in order to use this, I need to choose the target volume +print "Nchip: $Nchip\n"; +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; +print "Nfail: $Nfail\n"; + +my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail; +if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } + +# update the stats in the database +my $sql_update_stats = "UPDATE nights set"; +$sql_update_stats .= " Nchip = $Nchip"; +$sql_update_stats .= ", Ngood = $Ngood"; +$sql_update_stats .= ", Ndead = $Ndead"; +$sql_update_stats .= ", Ngone = $Ngone"; +$sql_update_stats .= ", Nsing = $Nsing"; +$sql_update_stats .= ", Ndups = $Ndups"; +$sql_update_stats .= ", Nsbck = $Nsbck"; +$sql_update_stats .= ", Nxbck = $Nxbck"; +$sql_update_stats .= ", Ncul1 = $Ncul1"; +$sql_update_stats .= ", Ncul2 = $Ncul2"; +$sql_update_stats .= ", Ncul3 = $Ncul3"; +$sql_update_stats .= ", Nfail = $Nfail"; +$sql_update_stats .= " WHERE dateobs = $dateword"; + +my $query = $raw_dbh->prepare($sql_update_stats); +$query->execute () or die "invalid response from RAW database\n"; exit 0; @@ -121,5 +157,5 @@ my @chip_ids; - open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; my @lines = ; @@ -160,8 +196,14 @@ if ($Nvalues != 3) { print "ERROR: invalid response?\n"; next; } + # data_state and possibly md5sum can be NULL, need to handle specially + my $dstate = $values[1]; if (not defined $dstate) { $dstate = "NULL"; } + my $md5sum = $values[2]; if (not defined $md5sum) { $md5sum = "XXXX"; } + push (@nebarray, $values[0]); - push (@ds_array, $values[1]); - push (@md_array, $values[2]); - if ($DEBUG > 4) { print "$values[0]\n $values[1]\n $values[2]\n"; } + push (@md_array, $md5sum); + push (@ds_array, $dstate); + + if ($DEBUG > 5) { print "V0: $values[0]\n"; print "V1: $values[1]\n"; print "V2: $values[2]\n"; } + if ($DEBUG > 4) { print "PRINT: $values[0]\n $dstate\n $md5sum\n"; } } @@ -178,10 +220,10 @@ # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) - my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z\.]*/(\S*)|; if ($DEBUG > 5) { print "nebname: $ext_id\n"; } my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); - + return ($rowref, $ext_id); } @@ -205,5 +247,5 @@ my $Nmain = 0; my $Nback = 0; - my $Ndead = 0; + my $Nlost = 0; # xattr = 0 means instance @ ITC (main) @@ -226,4 +268,8 @@ my $xattr = $values[3]; + if ($xattr > 1) { $Nlost ++; next; } # do not save instances on dead partitions + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + # count the number of each host by incrementing the hash element corresponding to the hostname: # NOTE: hostcount is global @@ -231,16 +277,13 @@ if (not defined $hostfile{$hostname}) { - open ($hostfile{$hostname}, ">$dateword/$hostname.inst.txt"); + open ($hostfile{$hostname}, ">$basedir/$dateword/$hostname.inst.$stage.txt"); ## XXX deal with open failure here } + my $fh = $hostfile{$hostname}; print $fh "$instance $data_st $md5_sum\n"; - - if ($xattr == 0) { $Nmain ++; } - if ($xattr == 1) { $Nback ++; } - if ($xattr > 1) { $Ndead ++; } - } - - return ($Nmain, $Nback, $Ndead); + } + + return ($Nmain, $Nback, $Nlost); } @@ -249,72 +292,167 @@ my $outfh = $_[0]; my $extID = $_[1]; - my $Nmain = $_[2]; - my $Nback = $_[3]; - my $Ndead = $_[4]; - - # SKIP : already fixed + my $data_st = $_[2]; + my $md5_sum = $_[3]; + my $Nmain = $_[4]; + my $Nback = $_[5]; + my $Nlost = $_[6]; + + my $outinfo = "$extID $data_st $md5_sum"; + $Nchip ++; + + my $haveMD5 = 0; + my $goodMD5 = 0; + if ($goodchips_r->{$extID}) { $haveMD5 = 1; $goodMD5 = 1; } + if ($failchips_r->{$extID}) { $haveMD5 = 1; $goodMD5 = 0; } + + if ($extID eq "gpc1/20100503/o5319g0003d/o5319g0003d.ota46.fits") { + print STDERR "$outinfo $Nmain $Nback $haveMD5 $goodMD5\n"; + } + + # GOOD : already fixed if (($Nmain == 1) && ($Nback == 1)) { - print $outfh "GOOD: $extID\n"; - return; - } - - # single copy @ ITC, no backup + print $outfh "GOOD: $outinfo\n"; + &update_counts (\$Ngood, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # SING : single copy @ ITC, no backup # make a backup if (($Nmain == 1) && ($Nback == 0)) { - print $outfh "SING: $extID\n"; - return; - } - - # ITC files with a single backup copies need to be culled - # cull Nmain - 1 - if (($Nmain > 1) && ($Nback == 1)) { - print $outfh "CULL: $extID\n"; - return; - } - - # multiple copies @ ITC + print $outfh "SING: $outinfo\n"; + &update_counts (\$Nsing, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # SBCK : single copy @ ATRC, no main copy + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print $outfh "SBCK: $outinfo\n"; + &update_counts (\$Nsbck, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # DUPS : multiple copies @ ITC, none at ATRC # make a backup and cull Nmain - 1 if (($Nmain > 1) && ($Nback == 0)) { - print $outfh "DUPS: $extID\n"; - return; - } - - # no more valid instances - if (($Nmain == 0) && ($Nback == 0) && $Ndead) { - print $outfh "DEAD: $extID\n"; - return; - } - if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { - print $outfh "GONE: $extID\n"; - return; - } - - # single copy @ ATRC - # make a backup @ ITC - if (($Nmain == 0) && ($Nback == 1)) { - print $outfh "SBCK: $extID\n"; - return; - } - - # multiple copies @ ATRC, but none at ITC + print $outfh "DUPS: $outinfo\n"; + &update_counts (\$Ndups, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # XBCK : multiple copies @ ATRC, but none at ITC # make a backup at ITC, cull ATRC extras if (($Nmain == 0) && ($Nback > 1)) { - print $outfh "XBCK: $extID\n"; + print $outfh "XBCK: $outinfo\n"; + &update_counts (\$Nxbck, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # DEAD : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && $Nlost) { + print $outfh "DEAD: $outinfo\n"; + &update_counts (\$Ndead, \$Nfail, $haveMD5, $goodMD5); + return; + } + # GONE : no more valid instances + if (($Nmain == 0) && ($Nback == 0) && ($Nlost == 0)) { + print $outfh "GONE: $outinfo\n"; + &update_counts (\$Ngone, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # CUL1 : multiple ITC files, single backup + # cull only ITC + if (($Nmain > 1) && ($Nback == 1)) { + print $outfh "CUL1: $outinfo\n"; + &update_counts (\$Ncul1, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # CUL2 : multiple ATRC files, single ITC + # cull only ATRC + if (($Nback > 1) && ($Nmain == 1)) { + print $outfh "CUL2: $outinfo\n"; + &update_counts (\$Ncul2, \$Nfail, $haveMD5, $goodMD5); + return; + } + + # CUL3 : excess ATRC & ITC files, cull from both? + # cull Nmain - 1 + if (($Nback > 1) && ($Nmain > 1)) { + print $outfh "CUL3: $outinfo\n"; + &update_counts (\$Ncul3, \$Nfail, $haveMD5, $goodMD5); return; } # unexpected state: - print $outfh "OOPS: $extID\n"; + print $outfh "OOPS: $outinfo\n"; return; } +sub update_counts { + + my $Nstat_r = $_[0]; + my $Nfail_r = $_[1]; + my $haveMD5 = $_[2]; + my $goodMD5 = $_[3]; + + ## if we know the MD5 is bad, add to Nfail, otherwise add to Ndups + if ($haveMD5 && !$goodMD5) { + $$Nfail_r ++; + } else { + $$Nstat_r ++; + } +} + +sub get_good_or_fail_chiplists { + + my $basedir = $_[0]; + my $dateword = $_[1]; + my $stage = $_[2]; + + # load the lists of good and fail chips + my $prev = $stage - 1; + my $goodfile = "$basedir/$dateword/goodchips.$prev.txt"; + my %goodchips; + if (-e $goodfile) { + open (GOODFILE, $goodfile); + my @lines = ; + close (GOODFILE); + + foreach my $line (@lines) { + chomp $line; + $goodchips{$line} = 1; + } + } + my $failfile = "$basedir/$dateword/failchips.$prev.txt"; + my %failchips; + if (-e $failfile) { + open (FAILFILE, $failfile); + my @lines = ; + close (FAILFILE); + + foreach my $line (@lines) { + chomp $line; + $failchips{$line} = 1; + } + } + return (\%goodchips, \%failchips); +} + sub parse_cmdopts { - my ($dateobs, $chiplist); - GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + # check that stage is an integer + unless (&is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -324,9 +462,12 @@ pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + my $fulldate = "$date_year/$date_month/$date_day"; - my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - return ($fulldate, $dateword, $chiplist) + my $yearobs = substr ($dateword, 0, 4); + my $basedir = "$topdir/$yearobs"; + + return ($fulldate, $dateword, $chiplist, $stage, $topdir, $basedir) } @@ -338,4 +479,5 @@ my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); + my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user ); if (-e $conffile) { @@ -355,8 +497,15 @@ if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /RAW_NAME /) { $rawdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_HOST /) { $rawdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_USER /) { $rawdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PASS /) { $rawdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PORT /) { $rawdb_port = (split /\s+/,$_)[2]; } } close(IN); print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; + print STDERR "raw dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n"; } @@ -373,4 +522,10 @@ pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name; + pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host; + pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user; + pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port; + pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass; + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } @@ -381,8 +536,15 @@ my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - return ($neb_dbh, $gpc_dbh); -} - - + my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host"; + if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; } + my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh, $gpc_dbh, $raw_dbh); +} + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + # example of using the return value of arrayref: Index: anches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl (revision 42530) +++ (revision ) @@ -1,423 +1,0 @@ -#! /usr/bin/env perl - -# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases - -# TESTRUN can be 1, 2, 3, 4 to get further in the tests -my $TESTRUN = 2; -my $DEBUG = 3; - -use DBI; -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -my ($tgthost,$tgtvol,$subdir,$movelist); - -# USAGE: check_chip_locations.pl --dateobs (night) --chip_id (chip_id) -# USAGE: check_chip_locations.pl --dateobs 2010.05.01 --chip_id XY33 - -my ($dateobs, $chip_id); -GetOptions( 'dateobs=s' => \$dateobs, 'chip_id=s' => \$chip_id) || pod2usage(2); - -my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); -my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); -my $conffile = 'nebulous.config'; - -# parse the nebulous.config file: -if (-e $conffile) { - open(IN,$conffile); - while() { - chomp; - if ($_ =~ /^\s*#/) { next; } # skip lines with leading # - if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } - if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } - - if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } - if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } - } - close(IN); - print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; - print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; -} - -pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; -pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; -pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; -pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; -pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; - -pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; -pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; -pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; -pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; -pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; - -pod2usage( -msg => "Cannot determine target date", -exitval => 2) unless defined $dateobs; -pod2usage( -msg => "Cannot determine target chip", -exitval => 2) unless defined $chip_id; - -# check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) - -my ($date_year, $date_month, $date_day) = split "/",$dateobs; -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); -pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); - -my $time = time(); - -my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; -if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } -my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - -my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; -if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } -my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - -# get the list of chip neb paths: -my @nebarray; -if (1) { - my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$date_year/$date_month/$date_day,00:00:00' and dateobs < '$date_year/$date_month/$date_day,23:59:59'"; - if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } - - my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); - my @tmparray = @$rowref; - my $Nrows = @tmparray; - - if ($DEBUG > 4) { print "found $Nrows chip images\n"; } - - foreach my $valref (@tmparray) { - my @values = @$valref; - # there should only be a single value (only uri selected above) - my $Nvalues = @values; - if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } - if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } - - push (@nebarray, $values[0]); - if ($DEBUG > 4) { print "$values[0]\n"; } - } -} - -# find the neb instances for each chip nebulous entry - -my %hostcount; - -foreach my $nebname (@nebarray) { - - # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits - # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits - # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) - my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; - if ($DEBUG > 5) { print "nebname: $ext_id\n"; } - - my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; - my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); - my @tmparray = @$rowref; - my $Nrows = @tmparray; - - if ($DEBUG > 5) { print "found $Nrows instances\n"; } - - if (0) { - foreach my $valref (@tmparray) { - my @values = @$valref; - # number of values should match list above (can this even fail?) - my $Nvalues = @values; - if ($Nvalues != 5) { print "ERROR: wrong number of fields?\n"; next; } - - if ($DEBUG > 2) { print "uri: $values[0], host: $values[1], avail: $values[2], xattr: $values[3], cab: $values[4]\n"; } - } - } - - # my $summary = "$ext_id $so_id"; - my $summary = "$ext_id"; - - # get user_copies for this storage_object - my $Nmain = 0; - my $Nback = 0; - my $Ndead = 0; - - # xattr = 0 means instance @ ITC (main) - # xattr = 1 means instance @ ATRC (back) - ## how many xattr = 0 and how many xattr = 1 - for (my $j = 0; $j < $Nrows; $j ++) { - my $valref = $tmparray[$j]; - my @values = @$valref; - - my $Nvalue = @values; - - if ($Nvalue != 5) { - print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; - next; - } - my $xattr = $values[3]; - my $hostname = $values[1]; - - # count the number of each host by incrementing the hash element corresponding to the hostname: - $hostcount{$hostname} ++; - - if ($xattr == 0) { $Nmain ++; } - if ($xattr == 1) { $Nback ++; } - if ($xattr > 1) { $Ndead ++; } - } - - # SKIP : already fixed - if (($Nmain == 1) && ($Nback == 1)) { - print STDOUT "GOOD: $summary\n"; - next; - } - - # single copy @ ITC, no backup - # make a backup - if (($Nmain == 1) && ($Nback == 0)) { - print STDOUT "SING: $summary\n"; - my $status = vsystem ("neb-replicate $ext_id"); - if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } - next; - } - - # ITC files with a single backup copies need to be culled - # cull Nmain - 1 - if (($Nmain > 1) && ($Nback == 1)) { - my $Ncull = $Nmain - 1; - print STDOUT "CULL: $summary\n"; - for (my $j = 0; $j < $Ncull; $j ++) { - my $status = vsystem("neb-cull $ext_id"); - if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } - } - next; - } - - # multiple copies @ ITC - # make a backup and cull Nmain - 1 - if (($Nmain > 1) && ($Nback == 0)) { - my $Ncull = $Nmain - 1; - print STDOUT "DUPS: $summary\n"; - my $status = vsystem("neb-replicate $ext_id"); - if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } - for (my $j = 0; $j < $Ncull; $j ++) { - $status = vsystem("neb-cull $ext_id"); - if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } - } - next; - } - - if (($Nmain == 0) && ($Nback == 0) && $Ndead) { - print STDOUT "DEAD: $summary\n"; - next; - } - if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { - print STDOUT "GONE: $summary\n"; - next; - } - - # single copy @ ATRC - # make a backup @ ITC - if (($Nmain == 0) && ($Nback == 1)) { - print STDOUT "SBCK: $summary\n"; - next; - } - - # multiple copies @ ATRC, but none at ITC - # make a backup at ITC, cull ATRC extras - if (($Nmain == 0) && ($Nback > 1)) { - print STDOUT "XBCK: $summary\n"; - next; - } - - # unexpected state: - print STDOUT "OOPS: $summary\n"; - next; -} - -my @hostnames = keys %hostcount; -foreach my $myname (@hostnames) { - print "host $myname = $hostcount{$myname}\n"; -} - -die "done"; - -#my $query = $dbh->prepare( $sql_get_volume_id ); - -#$query->execute( $tgt_volume ); -#my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; -#$query->finish; - - -### XX -### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; - -# load the list of files from movelist - -##XX open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; -##XX my @movefiles = ; -##XX close (MOVELIST); -##XX -##XX my $otime = $time; -##XX $time = time(); -##XX my $dtime = $time - $otime; -##XX -##XX for (my $i = 0; $i < @movefiles; $i++) { -##XX -##XX my $movefile = $movefiles[$i]; -##XX chomp $movefile; -##XX -##XX # the filename should have the following form: -##XX # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: -##XX # NN : 2-hexdigit name of the subsubdir -##XX # DDDDD : instance ID (long) -##XX # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name -##XX -##XX # I want to check two things: -##XX # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile -##XX # 2) does the vol_id of the instance_id match our vol_id ? -##XX -##XX unless (-e "$tgt_subdir/$movefile") { -##XX print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; -##XX next; -##XX } -##XX -##XX # print "movefile: $movefile\n"; -##XX -##XX my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; -##XX my $summary = "$tgt_subdir/$movefile"; -##XX -##XX if ( ! defined ($ins_id)) { -##XX print "ERROR : INVALID ID: $summary\n"; -##XX next; -##XX } -##XX -##XX if ( $ins_id =~ m/\D/ ) { -##XX print "ERROR : INVALID ID: $summary\n"; -##XX next; -##XX } -##XX $summary = "$summary : $ins_id"; -##XX -##XX # get so_id for this ins_id: -##XX my $so_id = 0; -##XX if (1) { -##XX my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; -##XX -##XX my $rowref = $dbh->selectall_arrayref( $so_id_query ); -##XX my @array = @$rowref; -##XX my $Nrows = @array; -##XX -##XX if ($Nrows == 0) { # We should find this instance -##XX print "ERROR : NO DB ENTRY: $summary\n"; -##XX next; -##XX } -##XX -##XX if ($Nrows > 1) { # We should only find one instance -##XX print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; -##XX next; -##XX } -##XX -##XX my $valref = $array[0]; -##XX my @values = @$valref; -##XX -##XX my $Nvalue = @values; -##XX -##XX if ($Nvalue != 1) { -##XX print "ERROR : INVALID DB RESPONSE: $summary\n"; -##XX next; -##XX } -##XX $so_id = $values[0]; -##XX } -##XX -##XX $summary = "$summary : $so_id"; -##XX -##XX # get user_copies for this storage_object -##XX my $Ncopies = 0; -##XX my $ext_id = ""; -##XX if (1) { -##XX my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; -##XX -##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); -##XX my @array = @$rowref; -##XX my $Nrows = @array; -##XX -##XX if ($Nrows == 0) { # user.copies is not set, skip -##XX print "SKIP: NO USER.COPIES ENTRY: $summary\n"; -##XX next; -##XX } -##XX -##XX if ($Nrows > 1) { # We should only find one instance -##XX print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; -##XX next; -##XX } -##XX -##XX my $valref = $array[0]; -##XX my @values = @$valref; -##XX -##XX my $Nvalue = @values; -##XX -##XX if ($Nvalue != 2) { -##XX print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; -##XX next; -##XX } -##XX $ext_id = $values[0]; -##XX $Ncopies = $values[1]; -##XX } -##XX -##XX $summary = "$summary : $ext_id"; -##XX -##XX if ($Ncopies < 2) { -##XX print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; -##XX next; -##XX } -##XX -##XX # check for backup instances -##XX if (1) { -##XX my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; -##XX -##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); -##XX my @array = @$rowref; -##XX my $Nrows = @array; -##XX -##XX if ($Nrows == 0) { # We should find this instance -##XX print "REPL: NO BACKUP COPY: $summary\n"; -##XX next; -##XX } -##XX -##XX ## my $valref = $array[0]; -##XX ## my @values = @$valref; -##XX ## -##XX ## my $Nvalue = @values; -##XX ## -##XX ## if ($Nvalue != 1) { -##XX ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; -##XX ## next; -##XX ## } -##XX } -##XX -##XX } # END loop over movefiles - -# example of using the return value of arrayref: - -# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); -# my @array = @$rowref; -# my $Nrows = @array; - -# print "NROWS: $Nrows\n"; - -# foreach my $valref (@array) { -# my @values = @$valref; -# foreach my $value (@values) { -# print "VAL: $value, "; -# } -# print "\n"; -# } - -sub vsystem { - # print STDERR "@_\n"; - return 0; - #my $status = system ("@_"); - #return $status; -} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_md5s_dateobs.pl (revision 42531) @@ -1,5 +1,5 @@ #! /usr/bin/env perl -my $DEBUG = 4; +my $DEBUG = 1; my $time = time(); @@ -12,61 +12,62 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: check_md5s_dateobs.pl --dateobs (night) - -my ($dateword) = &parse_cmdopts; - -my @instlist = glob ("$dateword/*.inst.txt"); -my @md5slist = glob ("$dateword/*.md5s.txt"); - -foreach my $instfile (@instlist) { - - # find the matching md5sum file for this host - - my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.txt|; - # print STDERR "host: $host\n"; - - my $md5match = "NONE"; - foreach my $md5sfile (@md5slist) { - - my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|; - # print STDERR "md5host: $md5shost\n"; - - if ($host ne $md5shost) { next; } - - $md5match = $md5sfile; - last; - } - - if ($md5match eq "NONE") { - print STDERR "missing md5sum file for $instfile\n"; - next; - } +# USAGE: check_md5s_dateobs.pl --dateobs (night) --topdir (topdir) + +my ($dateword, $stage, $topdir, $basedir) = &parse_cmdopts; +# basedir used only for night-level files + +my $instlist_r = &load_instlist_filenames ($basedir, $dateword, $stage); +my $md5slist_r = &load_md5slist_filenames ($basedir, $dateword, $stage, $instlist_r); + +my $Nfail = 0; +my %failchips; +my %goodchips; + +# make a list of the master instance files: + +foreach my $instfile (@$instlist_r) { + + # print "inst: $instfile\n"; + + my $md5sfile = $md5slist_r->{$instfile}; ## read in the md5sums - open (FILE, $md5match); + open (FILE, $md5sfile); my @md5data = ; close (FILE); - ## read in the list of instances - open (FILE, $instfile); - my @instdata = ; - close (FILE); + my $instdata_r = &load_instance_list ($basedir, $dateword, $stage, $instfile); # if these do not match, complain and move on + # we are allowed to have more MD5s than instances: some instances + # may have been culled during this process (but the MD5s stay behind) my $N1 = @md5data; - my $N2 = @instdata; - if ($N1 != $N2) { - print STDERR "inconsistent number of lines in $md5match : $N1 vs $N2\n"; + my $N2 = @$instdata_r; + if ($N1 < $N2) { + print STDOUT "not enough MD5s $md5sfile : $N1 vs $N2\n"; + $Nfail ++; next; - } - - # check for mismatched MD5sums - foreach my $line (@md5data) { - + } else { + # print STDOUT "checking MD5s $md5sfile : $N1 vs $N2\n"; + } + + # set up an empty hash for the instance names: + my %instfound = (); + foreach my $line (@$instdata_r) { chomp $line; my @words = split (/\s+/,$line); + my $inst = $words[0]; + $instfound{$inst} = 0; + } + + # check for mismatched MD5sums + # we are not checking that all instances in the inst file are in the MD5 file + foreach my $line (@md5data) { + + chomp $line; + my @words = split (/\s+/,$line); # we expect lines of the form: INST STATE MDtru MDobs - if (@words != 4) { die "error in MD5 list $md5match, line: $line\n"; } + if (@words != 4) { die "error in MD5 list $md5sfile, line: $line\n"; } my $inst = $words[0]; @@ -75,16 +76,170 @@ my $md5obs = $words[3]; + unless (defined $instfound{$inst}) { print "skip md5 for removed instance $inst\n"; next; } + $instfound{$inst} = 1; + if ($md5tru ne $md5obs) { - print STDERR "FAIL: $inst $state $md5tru $md5obs\n"; + &save_failure ($inst); + if ($DEBUG > 2) { print STDOUT "FAIL: $inst $state $md5tru $md5obs\n"; } + } else { + &save_goodchip ($inst); + if ($DEBUG > 2) { print STDOUT "$inst $state $md5tru = $md5obs\n"; } } } + + my @instfoundKEYS = keys %instfound; + foreach my $inst (@instfoundKEYS) { + unless ($instfound{$inst}) { die "missing MD5 for instance $inst\n"; } + } +} + +# save both good and fail chip lists +open (OUTFILE, ">$basedir/$dateword/goodchips.$stage.txt"); +my @goodchipURIs = keys %goodchips; +foreach my $uri (@goodchipURIs) { + print OUTFILE "$uri\n"; +} +close (OUTFILE); + +open (OUTFILE, ">$basedir/$dateword/failchips.$stage.txt"); +my @failchipURIs = keys %failchips; +foreach my $uri (@failchipURIs) { + print OUTFILE "$uri\n"; +} +close (OUTFILE); + + +## we only report an error if we could not find the md5sum files or if the inst and md5s +## files are inconsistent. In that case, we cannot check the md5s for all instances +## if the md5sum is wrong for a chip, mark that chip and save it. It will be skipped in fix_chip_locations +if ($Nfail) { + print STDOUT "ERROR: $Nfail errors\n"; + exit 1; +} else { + print STDOUT "SUCCESS: $Nfail errors\n"; + exit 0; +} + +# use global $basedir, $dateword, $myhost, $stage +sub load_instance_list { + my $basedir = $_[0]; + my $dateword = $_[1]; + my $stage = $_[2]; + my $instfile = $_[3]; + + my @instdata_list; + + # load the master list + &load_instance_list_onefile ($instfile, \@instdata_list); + + # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE + my ($myhost) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|; + my @instlist = glob ("$basedir/$dateword/$myhost.XY??.inst.$stage.txt"); + foreach my $infile (@instlist) { + &load_instance_list_onefile ($infile, \@instdata_list); + } + return (\@instdata_list); +} + +# read the instance info +sub load_instance_list_onefile { + my $infile = $_[0]; + my $instdata_list_r = $_[1]; # reference to the file data + + open(FILE,$infile) or die "cannot open file $infile\n"; + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + push (@$instdata_list_r, $line); + } +} + +# return only the master instlist filenames (ignore ones with the chipnames in the name) +sub load_instlist_filenames { + + my $basedir = $_[0]; + my $dateword = $_[1]; + my $stage = $_[2]; + + my @instlist_all = glob ("$basedir/$dateword/*.inst.$stage.txt"); + + my @instlist = (); + + foreach my $instfile (@instlist_all) { + + # skip the per-chip inst files (these should probably be merged into the master) + if ($instfile =~ m|XY[0-9][0-9].inst|) { next; } + + push (@instlist, $instfile); + } + return (\@instlist); +} + +# return a hash of the matched md5 file for each inst file +sub load_md5slist_filenames { + + my $basedir = $_[0]; + my $dateword = $_[1]; + my $stage = $_[2]; + my $instlist_r = $_[3]; + + my @md5slist = glob ("$basedir/$dateword/*.md5s.txt"); + + my %md5shash; + + my $Nfail = 0; + + foreach my $instfile (@$instlist_r) { + + # find the matching md5sum file for this host + + my ($host) = $instfile =~ m|$dateword/([0-9a-zA-Z]*).inst.$stage.txt|; + # print STDERR "host: $host\n"; + + my $md5match = "NONE"; + foreach my $md5sfile (@md5slist) { + + my ($md5shost) = $md5sfile =~ m|$dateword/([0-9a-zA-Z]*).md5s.txt|; + + if ($host ne $md5shost) { next; } + + # print STDERR "md5host: $md5shost : $md5sfile = $instfile\n"; + + $md5match = $md5sfile; + last; + } + + if ($md5match eq "NONE") { + print STDOUT "missing md5sum file for $instfile\n"; + $Nfail ++; + next; + } + $md5shash{$instfile} = $md5match; + + # print "match: $instfile = $md5match\n"; + + } + if ($Nfail > 0) { + print STDOUT "ERROR: $Nfail errors\n"; + exit 1; + } + return (\%md5shash); } sub parse_cmdopts { - my ($dateobs); - GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2); + my ($dateobs, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -98,5 +253,8 @@ my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - return ($dateword); + my $yearobs = substr ($dateword, 0, 4); + my $basedir = "$topdir/$yearobs"; + + return ($dateword, $stage, $topdir, $basedir); } @@ -107,2 +265,42 @@ return $status; } + +sub neb_reverse { + + my (@neb_path) = split /\//, $_[0]; + my $neb_key = $neb_path[-1]; + + # Remove ins_id + $neb_key =~ s/^\d+\.//; + + # Convert colons to slashes. + $neb_key =~ s%:%/%g; + + # strip ending .fz added for compressed .fits images + $neb_key =~ s%.fz$%%; + return $neb_key; +} + +sub save_failure { + my $nebfile = &neb_reverse($_[0]); + if (defined $failchips{$nebfile}) { + $failchips{$nebfile} ++; + } else { + $failchips{$nebfile} = 1; + } + return; +} + +sub save_goodchip { + my $nebfile = &neb_reverse($_[0]); + if (defined $goodchips{$nebfile}) { + $goodchips{$nebfile} ++; + } else { + $goodchips{$nebfile} = 1; + } + return; +} + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_night_status.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_night_status.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/check_night_status.pl (revision 42531) @@ -0,0 +1,146 @@ +#! /usr/bin/env perl +# this program must be run at the top-level rawfix directory + +my $DEBUG = 4; +my $time = time(); + +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_night_status.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist, $stage) = &parse_cmdopts; +my $yearobs = substr ($dateword, 0, 4); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$yearobs/$dateword/$chip_id.uris.$stage.txt"; + unless (-e $urifile) { die "missing chip URI list $urifile\n"; } + + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + + # these are done in stage 1: + if ($state eq "SING:") { $Nsing++; next; } + if ($state eq "SBCK:") { $Nsbck++; next; } + + # these are done in stage 2: + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # these are done in stage 3: + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; + +exit 0; + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage); +} + + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/create.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/create.rawfix.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/create.rawfix.sh (revision 42531) @@ -1,3 +1,6 @@ #!/bin/csh + +echo "do not use without caution" +exit 2 # make the rawfix database to track rawfix operations @@ -5,9 +8,29 @@ set DBUSER = "dvo" +# add in the iteration and the state summary + +mysql -h localhost -u root < qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_failures.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) +# USAGE: fix_chip_failures.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt --stage 0 + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; + + +my (@instance, @data_state, @md5tru, @md5obs) = &load_failures; + +sub load_failures { + + my (@instance, @data_state, @md5tru, @md5obs); + + # given a night, load the failures from log.md5chk.v0 + open (INLOG, "$topdir/$dateword/log.md5chk.$stage"); + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + unless ($line =~ /^FAIL:/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 5) { die "error in log output $line\n"; } + + push (@instance, $words[1]); + push (@data_state, $words[2]); + push (@md5tru, $words[3]); + push (@md5obs, $words[4]); + } + close (CHIPS); + + return (@instance, @data_state, @md5tru, @md5obs); +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir) +} + +# identify the + +sub neb_reverse { + + my (@neb_path) = split /\//, $file; + my $neb_key = $neb_path[-1]; + # Remove ins_id + $neb_key =~ s/^\d+\.//; + # Convert colons to slashes. + $neb_key =~ s%:%/%g; + $neb_key =~ s%.fz$%%; + print "$neb_key\n"; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/fix_chip_locations.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42531) @@ -12,5 +12,5 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) +# USAGE: fix_chip_locations.pl --dateobs (night) --stage (0,1,etc) [--chiplist (file) | --chipID (ID)] # we need to select targets for ITC copies of files currently only at ATRC: @@ -18,17 +18,17 @@ # for each chip, target a single host? -my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; -my ($neb_dbh) = &parse_db_config ('nebulous.config'); - -# read the list of chip_ids: -my @chip_ids = &load_chip_list ($chiplist); +my ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir, $basedir) = &parse_cmdopts; +my ($neb_dbh, $raw_dbh) = &parse_db_config ("$topdir/nebulous.config"); +# topdir used for nebulous.config and chiplist.txt, basedir used for night-level data # choose ITC machines on which to place backup copies: -my @volarray = &get_neb_volumes; -my $volnum = 0; +my @vol_copy = &get_neb_volumes_copy; # array of neb hosts to copy data to at ITC +my %vol_good = &get_neb_volumes_good; # hash of valid neb hosts + +my $Nchip = 0; # counts the number of instances my $Ngood = 0; +my $Ndead = 0; my $Ngone = 0; -my $Ndead = 0; my $Nsing = 0; my $Ndups = 0; @@ -36,10 +36,80 @@ my $Nxbck = 0; +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; +my $Nfail = 0; + +my ($goodchips_r, $failchips_r) = &get_good_or_fail_chiplists ($basedir, $dateword, $stage); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +my $volnum = 0; # select the volume to be used from the list of @vol_copy + +# loop over all chip_ids so we can distribute the target volumes foreach my $chip_id (@chip_ids) { - - my $urifile = "$dateword/$chip_id.uris.txt"; - open (FILE, "$urifile"); - my @lines = ; - close (FILE); + if (not defined $chipID or ($chipID eq $chip_id)) { + &fix_chip_locations_chipID ($chip_id, $volnum); + } + $volnum ++; + if ($volnum > @vol_copy) { $volnum = 0; } +} + +my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail; +if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } + +if (not defined $chipID) { + # update the stats in the database + my $sql_update_stats = "UPDATE nights set"; + $sql_update_stats .= " Nchip = $Nchip"; + $sql_update_stats .= ", Ngood = $Ngood"; + $sql_update_stats .= ", Ndead = $Ndead"; + $sql_update_stats .= ", Ngone = $Ngone"; + $sql_update_stats .= ", Nsing = $Nsing"; + $sql_update_stats .= ", Ndups = $Ndups"; + $sql_update_stats .= ", Nsbck = $Nsbck"; + $sql_update_stats .= ", Nxbck = $Nxbck"; + $sql_update_stats .= ", Ncul1 = $Ncul1"; + $sql_update_stats .= ", Ncul2 = $Ncul2"; + $sql_update_stats .= ", Ncul3 = $Ncul3"; + $sql_update_stats .= ", Nfail = $Nfail"; + $sql_update_stats .= " WHERE dateobs = $dateword"; + + my $query = $raw_dbh->prepare($sql_update_stats); + $query->execute () or die "invalid response from RAW database\n"; +} else { + + open (OUTFILE, ">$basedir/$dateword/$chipID.stats.$stage.txt"); + + print OUTFILE "Nchip: $Nchip\n"; + print OUTFILE "Ngood: $Ngood\n"; + print OUTFILE "Ngone: $Ngone\n"; + print OUTFILE "Ndead: $Ndead\n"; + print OUTFILE "Nsing: $Nsing\n"; + print OUTFILE "Ndups: $Ndups\n"; + print OUTFILE "Nsbck: $Nsbck\n"; + print OUTFILE "Nxbck: $Nxbck\n"; + print OUTFILE "Ncul1: $Ncul1\n"; + print OUTFILE "Ncul2: $Ncul2\n"; + print OUTFILE "Ncul3: $Ncul3\n"; + print OUTFILE "Nfail: $Nfail\n"; + + close (OUTFILE); +} + +exit 0; + +sub fix_chip_locations_chipID { + + my $chip_id = $_[0]; + my $volnum = $_[1]; + + # read from chip_id file, update to next stage number + my $urifile = "$basedir/$dateword/$chip_id.uris.$stage.txt"; + if (! -e $urifile) { die "missing file $urifile\n"; } + open (URIFILE, $urifile); + my @lines = ; + close (URIFILE); foreach my $line (@lines) { @@ -49,17 +119,26 @@ # we expect lines of the form: STATE PATH - # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits - if (@words != 2) { die "error in URI list $urifile, line: $line\n"; } + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits DataState MD5sum + if (@words != 4) { die "error in URI list $urifile, line: $line\n"; } my $state = $words[0]; my $ext_id = $words[1]; - + my $datast = $words[2]; + my $md5sum = $words[3]; + + my $haveMD5 = 0; + my $goodMD5 = 0; + if ($goodchips_r->{$ext_id}) { $haveMD5 = 1; $goodMD5 = 1; } + if ($failchips_r->{$ext_id}) { $haveMD5 = 1; $goodMD5 = 0; } + + $Nchip ++; + + # nothing to be done, or need be done for these: if ($state eq "GOOD:") { $Ngood++; next; } if ($state eq "DEAD:") { $Ndead++; next; } if ($state eq "GONE:") { $Ngone++; next; } - if ($state eq "DUPS:") { $Ndups++; next; } - if ($state eq "XBCK:") { $Nxbck++; next; } # one copy at ITC: neb-replicate will create a copy at ATRC + # does not depend on existence of MD5sum if ($state eq "SING:") { my $status = &vsystem ("neb-replicate $ext_id"); @@ -67,6 +146,6 @@ my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } - open (OUTFILE, ">>$dateword/$newhost.inst.txt"); - print OUTFILE "$newfile\n"; + open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; close (OUTFILE); $Nsing ++; @@ -75,29 +154,100 @@ # one copy at ATRC: neb-replicate will create a copy at ITC iff we supply the target volume + # does not depend on existence of MD5sum if ($state eq "SBCK:") { - my $tgtvol = $volarray[$volnum]; + my $tgtvol = $vol_copy[$volnum]; my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); if ($status) { print STDERR "FAIL SBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } - open (OUTFILE, ">>$dateword/$newhost.inst.txt"); - print OUTFILE "$newfile\n"; + open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; close (OUTFILE); $Nsbck ++; next; } - } - $volnum ++; -} - -print "Ngood: $Ngood\n"; -print "Ngone: $Ngone\n"; -print "Ndead: $Ndead\n"; -print "Nsing: $Nsing\n"; -print "Ndups: $Ndups\n"; -print "Nsbck: $Nsbck\n"; -print "Nxbck: $Nxbck\n"; - -exit 0; + + # multiple copies @ ITC, but none at ATRC. use neb-replicate to make copies + # skip if we do not know the state of the md5sum + if ($state eq "DUPS:") { + if ($haveMD5 && $goodMD5) { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL DUPS: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + } + ## if we know the MD5 is bad, add to Nfail, otherwise add to Ndups + if ($haveMD5 && !$goodMD5) { + $Nfail ++; + } else { + $Ndups++; + } + next; + } + + # multiple copies @ ATRC, but none @ ITC: neb-replicate will create a copy at ITC iff we supply the target volume + # skip if we do not know the state of the md5sum + if ($state eq "XBCK:") { + if ($haveMD5 && $goodMD5) { + my $tgtvol = $vol_copy[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL XBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$basedir/$dateword/$newhost.$chip_id.inst.$stage.txt"); + print OUTFILE "$newfile $datast $md5sum\n"; + close (OUTFILE); + } + ## if we know the MD5 is bad, add to Nfail, otherwise add to Nxbck: + if ($haveMD5 && !$goodMD5) { + $Nfail ++; + } else { + $Nxbck++; + } + next; + } + + # one copy at ITC: neb-replicate will create a copy at ATRC + # skip if we do not know the state of the md5sum + if (($state eq "CUL1:") || ($state eq "CUL2:") || ($state eq "CUL3:")) { + + if ($haveMD5 && $goodMD5) { + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($ext_id); + + my ($main_keep, $back_keep, $cull_list_r) = &check_instance_xattr ($inforows, $ext_id); + + if ($main_keep eq "") { print STDERR "FAIL $state: missing ITC copy for $ext_id\n"; next; } + if ($back_keep eq "") { print STDERR "FAIL $state: missing ATRC copy for $ext_id\n"; next; } + + my $NcullTotal = @$cull_list_r; + foreach my $instance (@$cull_list_r) { + print "state $state : checking instance $instance ... "; + my ($volname) = $instance =~ m|file:///data/([0-9a-zA-Z]*\.[0-9])/\S*|; + print "volname $volname\n"; + unless (defined $vol_good{$volname}) { die "ERROR: invalid volume name $volname\n"; } + # print STDERR "neb-cull $state: $ext_id $volname ($main_keep $back_keep)\n"; + my $status = &vsystem ("neb-cull --volume $volname $ext_id"); + if ($status) { print STDERR "FAIL CULL: neb-replicate : $ext_id\n"; next; } + } + # print STDERR "Ncull Total $NcullTotal\n"; + } + + ## if we know the MD5 is bad, add to Nfail, otherwise add to NculX + if ($haveMD5 && !$goodMD5) { + $Nfail ++; + } else { + if ($state eq "CUL1:") { $Ncul1++; next; } + if ($state eq "CUL2:") { $Ncul2++; next; } + if ($state eq "CUL3:") { $Ncul3++; next; } + } + } + } +} sub get_new_instance { @@ -120,6 +270,97 @@ } -# get the list of chip neb paths for one chip -sub get_neb_volumes { +sub get_chip_instances { + + my $ext_id = $_[0]; + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# select 1 instance with xattr = 0, 1 with xattr = 1. +# the rest should be culled. + +sub check_instance_xattr { + + my $rowref = $_[0]; + my $ext_id = $_[1]; + + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $main_keep = ""; + my $back_keep = ""; + my @cull_list = (); + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + # the number of returned values per row must match the query above (in get_chip_instances) + my $Nvalue = @values; + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + if (($xattr == 0) && ($main_keep eq "")) { $main_keep = $instance; next; } + if (($xattr == 1) && ($back_keep eq "")) { $back_keep = $instance; next; } + + if ($xattr > 1) { print "ignore lost instance $instance\n"; next; } + + # if we make it here, we already have an instance at this location + push (@cull_list, $instance); + } + return ($main_keep, $back_keep, \@cull_list); +} + +sub get_good_or_fail_chiplists { + + my $basedir = $_[0]; + my $dateword = $_[1]; + my $stage = $_[2]; + + # load the lists of good and fail chips + my $prev = $stage - 1; + my $goodfile = "$basedir/$dateword/goodchips.$prev.txt"; + my %goodchips; + if (-e $goodfile) { + open (GOODFILE, $goodfile); + my @lines = ; + close (GOODFILE); + + foreach my $line (@lines) { + chomp $line; + $goodchips{$line} = 1; + } + } + my $failfile = "$basedir/$dateword/failchips.$prev.txt"; + my %failchips; + if (-e $failfile) { + open (FAILFILE, $failfile); + my @lines = ; + close (FAILFILE); + + foreach my $line (@lines) { + chomp $line; + $failchips{$line} = 1; + } + } + return (\%goodchips, \%failchips); +} + +# get the list of neb hosts to copy data to at ITC +sub get_neb_volumes_copy { my @volarray; @@ -148,4 +389,32 @@ } +# get the list of valid neb hosts (ITC or ATRC) +sub get_neb_volumes_good { + + my %volarray; + + my $sql_neb_volumes = "select name from mountedvol where ((xattr = 0) or (xattr = 1))"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + $volarray{$values[0]} = 1; + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return %volarray; +} + # read the list of chip_ids: sub load_chip_list { @@ -154,5 +423,5 @@ my @chip_ids; - open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; my @lines = ; @@ -173,10 +442,16 @@ sub parse_cmdopts { - my ($dateobs, $chiplist); - GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + my ($dateobs, $chiplist, $chipID, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'chipID=s' => \$chipID, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; - + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) @@ -190,5 +465,8 @@ my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - return ($fulldate, $dateword, $chiplist); + my ($yearobs) = substr ($dateword, 0, 4); + my $basedir = "$topdir/$yearobs"; + + return ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir, $basedir); } @@ -199,4 +477,5 @@ my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user ); if (-e $conffile) { @@ -210,7 +489,14 @@ if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /RAW_NAME /) { $rawdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_HOST /) { $rawdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_USER /) { $rawdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PASS /) { $rawdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PORT /) { $rawdb_port = (split /\s+/,$_)[2]; } } close(IN); print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "raw dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n"; } @@ -221,15 +507,30 @@ pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name; + pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host; + pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user; + pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port; + pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass; + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; - return ($neb_dbh); + my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host"; + if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; } + my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh, $raw_dbh); } sub vsystem { print STDERR "@_\n"; - #return 0; + # return 0; my $status = system ("@_"); return $status; } + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh (revision 42531) @@ -0,0 +1,37 @@ +#!/bin/csh + +if ($#argv != 3) then + echo "USAGE: get_hosts_fixchip.sh (topdir) (dateobs) (iter)" + exit 2 +endif + +set topdir = $1 +set dateobs = $2 +set iter = $3 + +if (! -d $topdir) then + echo "ERROR: missing top-level directory $topdir" + exit 3 +endif + +set yearobs = `echo $dateobs | awk '{print substr($1,1,4)}'` +set basedir = $topdir/$yearobs + +if (! -d $basedir/$dateobs) then + echo "ERROR: missing dateobs directory $basedir/$dateobs" + exit 3 +endif + +# report both the dateobs and the hostnames +echo "dateobs chipID iter fixhost" + +foreach f ($basedir/$dateobs/XY??.hosts.txt) + set myhost = `cat $f | grep -v ippb | sort -k 2n | tail -n 1 | prcol 1` + set chipID = `echo $f | awk -F/ '{print $NF}' | awk -F. '{print $1}'` + if ("$myhost" == "") then + set myhost = ipp156 + endif + echo $dateobs $chipID $iter $myhost +end + +## XXX ipp156 is hardwired for no good reason Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_hosts_md5s.sh (revision 42531) @@ -1,17 +1,27 @@ #!/bin/csh -if ($#argv != 1) then - echo "USAGE: get_hosts_md5s.sh (dateobs)" +if ($#argv != 3) then + echo "USAGE: get_hosts_md5s.sh (topdir) (dateobs) (stage)" exit 2 endif -set dirname = $1 +set topdir = $1 +set dateobs = $2 +set stage = $3 -if (! -d $dirname) then - echo "ERROR: missing dateobs directory $dirname" +if (! -d $topdir) then + echo "ERROR: missing top-level directory $topdir" + exit 3 +endif + +set yearobs = `echo $dateobs | awk '{print substr($1,1,4)}'` +set basedir = $topdir/$yearobs + +if (! -d $basedir/$dateobs) then + echo "ERROR: missing dateobs directory $basedir/$dateobs" exit 3 endif # report both the dateobs and the hostnames -ls $dirname/*.inst.txt | awk -v dateobs=$dirname -F/ '(NR == 1){print "dateobs md5host"}{print dateobs, $2}' | sed s/.inst.txt// +ls $basedir/$dateobs/*.inst.$stage.txt | grep -v "XY...inst" | awk -v dateobs=$dateobs -v stage=$stage -F/ '(NR == 1){print "dateobs stage md5host"}{print dateobs, stage, $NF}' | sed s/.inst.$stage.txt// exit 0 Index: anches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl (revision 42530) +++ (revision ) @@ -1,50 +1,0 @@ -#! /usr/bin/env perl - -die "XXX not done yet"; - -my $DEBUG = 4; -my $time = time(); - -use DBI; -use Carp; -use Pod::Usage qw( pod2usage ); - -use strict; -use warnings FATAL => qw( all ); -use Getopt::Long qw( GetOptions :config auto_help auto_version ); - -# USAGE: get_md5s_dateobs.pl --dateobs (night) - -my ($fulldate, $dateword) = &parse_cmdopts; - -# first, ensure the dateobs directory is - - - -sub parse_cmdopts { - - my ($dateobs); - GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2); - - pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; - - # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) - - my ($date_year, $date_month, $date_day) = split "/",$dateobs; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); - pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); - - my $fulldate = "$date_year/$date_month/$date_day"; - my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - - return ($fulldate, $dateword); -} - -sub vsystem { - print STDERR "@_\n"; - #return 0; - my $status = system ("@_"); - return $status; -} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_md5s_instances.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_md5s_instances.pl (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/get_md5s_instances.pl (revision 42531) @@ -4,5 +4,4 @@ my $time = time(); -use DBI; use Carp; use Pod::Usage qw( pod2usage ); @@ -12,7 +11,8 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) --stage (0,1,etc) -my ($myhost, $input) = &parse_cmdopts; +my $input; +my ($myhost, $dateword, $topdir, $basedir, $stage) = &parse_cmdopts; # check that we are running this script on 'myhost' @@ -21,17 +21,15 @@ # read the list of instances: -my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list ($input); +my ($ins_list_r, $dst_list_r, $md5_list_r) = &load_instance_list (); -my $output = $input; -$output =~ s/inst/md5s/; +my $output = "$basedir/$dateword/$myhost.md5s.txt"; -if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } +# read in existing md5s +my $out_hash_r; if (-e $output) { - # check that all instances are in the output file? - print "$output exists, skipping\n"; - exit 0; + $out_hash_r = &load_output_list ($output); } -open (OUTFILE, ">$output"); +open (OUTFILE, ">>$output"); # foreach my $instance (@$ins_list_r) { @@ -42,13 +40,23 @@ my $md5sum_tru = $md5_list_r->[$i]; + # if the instance is already in the output file, skip it + if (defined $out_hash_r) { + my $name = $out_hash_r->{$instance}; + if (defined $name) { + print STDERR "skipping $instance (already in output)\n"; + next; + } + } + my $filename = $instance; $filename =~ s|file://||; - my $line; - if (0) { - $line = "fakevalue $filename"; # XXX TEST: disable calculation: - } else { - $line = `md5sum $filename`; + # check for the existence of the file first: + unless (-e $filename) { + print OUTFILE "$instance $data_state $md5sum_tru XXX\n"; + next; } + + my $line = `md5sum $filename`; my ($md5sum, $rawfile) = split (" ", $line); @@ -58,9 +66,26 @@ exit 0; -# read the list of chip_ids: +# use global $basedir, $dateword, $myhost, $stage sub load_instance_list { + my (@ins_list, @dst_list, @md5_list); + + # load the master list + my $input = "$basedir/$dateword/$myhost.inst.$stage.txt"; + &load_instance_list_onefile ($input, \@ins_list, \@dst_list, \@md5_list); + + # there may be additional instances in files with names like HOSTNAME.XYnn.inst.STAGE + my @instlist = glob ("$basedir/$dateword/$myhost.XY??.inst.$stage.txt"); + foreach my $infile (@instlist) { + &load_instance_list_onefile ($infile, \@ins_list, \@dst_list, \@md5_list); + } + return (\@ins_list, \@dst_list, \@md5_list); +} + +# read the instance info +sub load_instance_list_onefile { my $infile = $_[0]; - - my (@ins_list, @dst_list, @md5_list); + my $ins_list_r = $_[1]; # reference to the instance list array + my $dst_list_r = $_[2]; # reference to the instance list array + my $md5_list_r = $_[3]; # reference to the instance list array open(FILE,$infile) or die "cannot open file $infile\n"; @@ -74,22 +99,53 @@ if (@words != 3) { die "error in input $infile : $line\n"; } - push (@ins_list, $words[0]); - push (@dst_list, $words[1]); - push (@md5_list, $words[2]); + push (@$ins_list_r, $words[0]); + push (@$dst_list_r, $words[1]); + push (@$md5_list_r, $words[2]); + } + close (FILE); +} + +# read the instance list in the output file +sub load_output_list { + my $infile = $_[0]; + + my (%out_hash); + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 4) { die "error in input $infile : $line\n"; } + $out_hash{$words[0]} = 1; } close (FILE); - return (\@ins_list, \@dst_list, \@md5_list); + return (\%out_hash); } sub parse_cmdopts { - my ($myhost, $input); - GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input) || pod2usage(2); + my ($myhost, $topdir, $dateobs, $stage); + GetOptions( 'hostname=s' => \$myhost, 'topdir=s' => \$topdir, 'dateobs=s' => \$dateobs, 'stage=s' => \$stage) || pod2usage(2); - pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYYMMDD", -exitval => 2) unless defined $dateobs; +# pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; - return ($myhost, $input); + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + + my $yearobs = substr ($dateobs, 0, 4); + my $basedir = "$topdir/$yearobs"; + + return ($myhost, $dateobs, $topdir, $basedir, $stage); } @@ -100,2 +156,7 @@ return $status; } + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl (revision 42531) @@ -0,0 +1,278 @@ +#!/usr/bin/env perl + +# this program is used to run 'burntool' on images for a single chip +# for all exposures for a given time period. +# USAGE: ipp_apply_burntool.pl --dbname gpc1 --class_id XY00 --dateobs_begin YYYY/MM/DD --dateobs_end YYYY/MM/DD + +use warnings; +use strict; +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +use PS::IPP::Metadata::List qw( parse_md_list ); +use PS::IPP::Config 1.01 qw( :standard ); +use File::Temp qw( tempfile ); +use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); + +my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $dbname, $logfile, $verbose, $save_temps, $rerun, $psvidOutput); +GetOptions( + 'class_id=s' => \$class_id, # chip identifier + 'dateobs_begin=s' => \$dateobs_begin, # exposure date/time range start + 'dateobs_end=s' => \$dateobs_end, # exposure date/time range stop + 'dbname|d=s' => \$dbname, # Database name + 'psvidOutput|d=s' => \$psvidOutput, # output base for psvideo data + 'logfile=s' => \$logfile, + 'rerun' => \$rerun, + 'verbose' => \$verbose, # Print to stdout + 'save-temps' => \$save_temps, # Save temporary files? + + ) or pod2usage( 2 ); + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( + -msg => "Required options: --class_id --dateobs_begin --dateobs_end --dbname", + -exitval => 3, + ) unless + defined $class_id and + defined $dateobs_begin and + defined $dateobs_end and + defined $dbname; + +# burntool is only valid for GPC1, do not allow substitutes +my $camera = "GPC1"; + +my $missing_tools; +my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1); +my $funpack = can_run('funpack') or (warn "Can't find funpack" and $missing_tools = 1); +my $burntool = can_run('burntool') or (warn "Can't find burntool" and $missing_tools = 1); +my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); +my $nebXattr = can_run('neb-xattr') or (warn "Can't find neb-xattr" and $missing_tools = 1); +my $nebreplicate = can_run('neb-replicate') or (warn "Can't find neb-replicate" and $missing_tools = 1); +my $psvideostats = can_run('psvideostats.sh') or (warn "Can't find psvideostats.sh" and $missing_tools = 1); +if ($missing_tools) { + warn("Can't find required tools."); + exit($PS_EXIT_CONFIG_ERROR); +} + +# XXX check we can access / create directory named $psvidOutput + +my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files + +# IPP configuration (including nebulous) +my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up"); + +$ipprc->redirect_output($logfile) if $logfile; + +# Determine the value of a "good" burntool run. +# XXX this program should always generate the latest version +# but the processing system must be told to use an earlier version +# until the newer version is validated. Perhaps split these. +my $config_cmd = "$ppConfigDump -camera $camera -get-key BURNTOOL.STATE.GOOD"; +my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $config_cmd, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); +} + +my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", 0, 0, $class_id, $PS_EXIT_SYS_ERROR); + +my $burntoolStateGood = 999; +foreach my $cfg (@$recipeData) { + if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') { + $burntoolStateGood = $cfg->{value}; + } +} +if ($burntoolStateGood == 999) { + &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, $class_id, 0, $PS_EXIT_SYS_ERROR); +} +my $outState = -1 * abs($burntoolStateGood); + +print "Target burntool version: $burntoolStateGood\n"; + +# Define list of images to examine. +my $command = "$regtool -processedimfile"; +$command .= " -class_id $class_id"; +$command .= " -dateobs_begin $dateobs_begin"; +$command .= " -dateobs_end $dateobs_end"; +$command .= " -ordered_by_date"; +$command .= " -dbname $dbname" if defined $dbname; + +( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run ( command => $command, verbose => $verbose); +unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform regtool: $error_code", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); +} + +# EAM : I am not sure why this is implemented in an esoteric way rather than using the MD parsing API +my @whole = split /\n/, (join "", @$stdout_buf); +my @single = (); +my $files; +while ( scalar @whole > 0 ) { + my $value = shift @whole; + push @single, $value; + if ($value =~ /^\s*END\s*$/) { + push @single, "\n"; + + my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ); + &my_die("Unable to parse output from regtool", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_PROG_ERROR) unless + defined $list; + push @{ $files }, @$list; + @single = (); + } +} + +#my $metadata = $mdcParser->parse(join "", @$stdout_buf) or +# &my_die("Unable to parse metadata", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); +# +#my $files = parse_md_list($metadata) or +# &my_die("Unable to parse metadata list", $class_id, $dateobs_begin, $dateobs_end, $PS_EXIT_SYS_ERROR); + +my $REALRUN = 1; + +# Process each file in turn, checking to see if it needs processing, and doing it all in a single pass +my $processNext = 0; +my $previousTable = ''; + +foreach my $file (@$files) { + my $exp_id = $file->{exp_id}; + my $state = $file->{burntool_state}; + + my $rawImfile = $file->{uri}; + my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0); + + my $outTable = $file->{uri}; + + if($burntoolStateGood == 15) { + $outTable =~ s/fits$/burn.v15.tbl/; + } else { + $outTable =~ s/fits$/burn.tbl/; + } + my $outTableReal = $ipprc->file_resolve($outTable, 1); + + my $process = 0; + my $previousTableStyle = 0; + + if ($state == 0) { + $process = 1; + } + elsif ($state == -1) { + $process = 1; + } + elsif ($state == -3) { + $process = 1; + } + elsif ($state == -2) { + &my_die("Aborting as $rawImfile has modified pixel data!"); + } + elsif ($state == $burntoolStateGood) { + $process = 0; + $previousTableStyle = 1; + } + elsif ($state == -1 * $burntoolStateGood) { + $process = 0; + $previousTableStyle = -1; + } + else { + $process = 1; + } + if ($rerun) { + $process = 1; + } + if ($REALRUN == 0) { + print "##Pretending to process as instructed!\n"; + $process = 1; + } + if (($process == 1)||($processNext == 1)) { + my $status; + $processNext = 1; + + my $tempfile = new File::Temp ( TEMPLATE => "$file->{exp_name}.XXXX", + DIR => '/tmp/', + UNLINK => !$save_temps, + SUFFIX => '.fits'); + my $tempPixels = $tempfile->filename; + + # extract local copy of thie chip imfile + $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN); + if ($status) { + &my_die("failed on funpack",$exp_id,$class_id); + } + + # Run burntool + if ($previousTable ne '') { + $status = vsystem ("$burntool $tempPixels $previousTable out=$outTableReal tableonly=t persist=t", $REALRUN); + } else { + $status = vsystem ("$burntool $tempPixels out=$outTableReal tableonly=t persist=t", $REALRUN); + } + if ($status) { + &my_die("failed on burntool",$exp_id,$class_id); + } + &my_die("Unable to find output file: $outTableReal", $exp_id, $class_id) unless $ipprc->file_exists($outTableReal); + + $status = vsystem ("$nebXattr --write $outTable user.copies:2",$REALRUN); + $status = vsystem ("$nebreplicate --set_copies 2 $outTable",$REALRUN); + if ($status) { + &my_die("failed to replicate imfile"); + } + + # update database with new burntool state + $status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state $outState", $REALRUN); + if ($status) { + &my_die("failed to update imfile"); + } + + # generate psvideostats + my $psvidOutfile = "$psvidOutput/psvideostats.$class_id.dat"; + $status = vsystem ("$psvideostats $tempPixels $psvidOutfile $file->{exp_name} $class_id", $REALRUN); + if ($status) { + # die or warn? + &my_die("failed to update imfile"); + } + + $previousTableStyle = -1; + + print "\n"; + } + + if ($previousTableStyle == 1) { + $previousTable = "infits=$rawImfileReal"; + } + elsif ($previousTableStyle == -1) { + $previousTable = "in=$outTableReal"; + } + else { + die "previousTableStyle undefined!\n"; + } + +} + +exit 0; + +sub vsystem { + my $command = shift; + my $realrun = shift; + + print "$command\n"; + + my $status = 0; + if ($realrun) { + $status = system ($command); + } + + return $status; +} + +sub my_die { + my $message = shift; + if ($#_ != -1) { + my $exp_id = shift; + my $class_id = shift; + vsystem("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -burntool_state -3",1); + } + printf STDERR "$message\n"; + exit 1; +} Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/psvideostats.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/psvideostats.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/psvideostats.sh (revision 42531) @@ -0,0 +1,240 @@ +#!/usr/bin/env mana + +# psvideostats.sh (input.fits) (output.txt) +# if no video table in this chip, no output + +macro psvideostats.imfile + if ($0 != 5) + echo "USAGE: psvideostats.imfile (input) (output) (expname) (chipID)" + break + end + + local input output expname chipID + $input = $1 + $output = $2 + $expname = $3 + $chipID = $4 + + $HAS_VIDEO = 0 + + # define the header + echo "trying $input" + get.vidstats.imfile $input + + if (not($HAS_VIDEO)) return + + # chip & exposure info + keyword hdr FILENAME filename + keyword hdr FPPOS fppos + + output $output + echo "$expname $chipID $filename $fppos $VIDSTATS" + output stdout +end + +macro get.vidstats.imfile + if ($0 != 2) + echo "USAGE: get.vidstats.imfile (file)" + break + end + + local filename + $filename = $1 + + init.vidcells + + # to find the video cell, we can look at the PHU and grab CELLMODE + rd hdr $filename + keyword hdr CELLMODE cellmode + strstr -q $cellmode V -start Nstart -length Nchar + if ($Nstart == -1) + echo "no video data for this chip" + return + end + + sprintf extname "video_table_xy%02d" cellXY[$Nstart] + + # read the video cell table (need cnpix1,2 to measure flat-corrected flux) + data $filename + break -auto off + read -fits $extname frame_start_time frame_complete_time centroid_x centroid_y fwhm_x fwhm_y flux cnpix1 cnpix2 + if (not($STATUS)) + echo "missing FITS table field for $filename" + return + end + break -auto on + + $HAS_VIDEO = 1 + + # set default values for outputs + init.values + + # calculate the time offset + set dtime = frame_start_time - frame_start_time[0] + set dx = centroid_x - cnpix1 + set dy = centroid_y - cnpix2 + + # a good star measurement is within the window 15 < (dx,dy) 85 pixels . + + set good = (dtime > 4) && (dx > 15) && (dx < 85) && (dy > 15) && (dy < 85) && (dtime < dtime[-1] - 2.0) + subset dts = dtime where good + subset dxs = dx where good + subset dys = dy where good + subset flux_s = flux where good + $GOODFRAC = dxs[] / dx[] + + if (dxs[] == 0) + save.values + return + end + + # generate percentiles: (5, 16, 50, 84, 95) for dxs, dys + get.percentiles dxs; $DX_SIG_RAW = $dPCT_68 / 2.0; $DX_D90_RAW = $dPCT_90; $DX_MED_RAW = $PCT_50 + get.percentiles dys; $DY_SIG_RAW = $dPCT_68 / 2.0; $DY_D90_RAW = $dPCT_90; $DY_MED_RAW = $PCT_50 + vstat -q flux_s; $FLUX_MEDIAN = $MEDIAN; $FLUX_STDEV = $SIGMA + + # smoothed version of dxs, dys: + vmedfilt dxs dxsmooth 10 + vmedfilt dys dysmooth 10 + set seqsm = ramp(dxsmooth) + + # smoothed vector stdev + get.percentiles dxsmooth; $DX_SIG_SMT = $dPCT_68 / 2.0; $DX_D90_SMT = $dPCT_90; $DX_MED_SMT = $PCT_50 + get.percentiles dysmooth; $DY_SIG_SMT = $dPCT_68 / 2.0; $DY_D90_SMT = $dPCT_90; $DY_MED_SMT = $PCT_50 + + # measure long-term drift + vstat -q dxsmooth where (seqsm < 10); $DX_DRIFT_S = $MEDIAN + vstat -q dxsmooth where (seqsm > seqsm[-1] - 10); $DX_DRIFT_E = $MEDIAN + $DX_DRIFT = $DX_DRIFT_E - $DX_DRIFT_S + + vstat -q dysmooth where (seqsm < 10); $DY_DRIFT_S = $MEDIAN + vstat -q dysmooth where (seqsm > seqsm[-1] - 10); $DY_DRIFT_E = $MEDIAN + $DY_DRIFT = $DY_DRIFT_E - $DY_DRIFT_S + + # scatter out of min range + vstat -q dxsmooth where (abs(dxsmooth - $DX_MED_SMT) > 2.0); $FX_OUT_SMT = $NPIX / dxsmooth[] + vstat -q dysmooth where (abs(dysmooth - $DY_MED_SMT) > 2.0); $FY_OUT_SMT = $NPIX / dysmooth[] + + save.values +end + +#### Utilities #### + +macro init.values + $GOODFRAC = 0 + $FLUX_MEDIAN = NAN + $FLUX_STDEV = NAN + $DX_SIG_RAW = NAN + $DY_SIG_RAW = NAN + $DX_D90_RAW = NAN + $DY_D90_RAW = NAN + $DX_SIG_SMT = NAN + $DY_SIG_SMT = NAN + $DX_D90_SMT = NAN + $DY_D90_SMT = NAN + $DX_DRIFT = NAN + $DY_DRIFT = NAN + $FX_OUT_SMT = NAN + $FY_OUT_SMT = NAN +end + +# output values are all Global +macro save.values + sprintf VIDSTAT1 "%5.3f %9.1f %9.1f" $GOODFRAC $FLUX_MEDIAN $FLUX_STDEV + sprintf VIDSTAT2 "%6.2f %6.2f : %6.2f %6.2f" $DX_SIG_RAW $DY_SIG_RAW $DX_D90_RAW $DY_D90_RAW + sprintf VIDSTAT3 "%6.2f %6.2f : %6.2f %6.2f" $DX_SIG_SMT $DY_SIG_SMT $DX_D90_SMT $DY_D90_SMT + sprintf VIDSTAT4 "%6.2f %6.2f : %6.4f %6.4f" $DX_DRIFT $DY_DRIFT $FX_OUT_SMT $FY_OUT_SMT + sprintf VIDSTATS "%s : %s : %s : %s" "$VIDSTAT1" "$VIDSTAT2" "$VIDSTAT3" "$VIDSTAT4" +end + +# only used if needed +macro init.vidcells + if (cellXY?[]) return + + local ix iy cellXY + + # create a lookup table sequence -> xy + delete -q cellXY + for iy 0 8 + for ix 0 8 + $cellXY = $ix\$iy + concat $cellXY cellXY + end + end +end + +# set minRange - maxRange as 2 x (0.05 to 0.95) range +macro get.threshrange + if ($0 != 2) + echo "USAGE: get.threshrange (vector)" + break + end + + local vecname + $vecname = $1 + + vstat -q $vecname + histogram $vecname _tmpNx 0 {2*$MEDIAN} {0.01*$MEDIAN} -range _tmpdx + cumulative _tmpNx _tmpSx + set _tmpFx = _tmpSx / _tmpSx[-1] + threshold -saturate -q _tmpdx _tmpFx 0.05 + $minT = $threshold + + threshold -saturate -q _tmpdx _tmpFx 0.95 + $maxT = $threshold + + $allRange = $maxT - $minT + $midRange = 0.5*($maxT + $minT) + $minRange = $midRange - $allRange + $maxRange = $midRange + $allRange +end + +# set minRange - maxRange as 2 x (0.05 to 0.95) range +macro get.percentiles + if ($0 != 2) + echo "USAGE: get.percentiles (vector)" + break + end + + local vecname + $vecname = $1 + + # percentiles to calculate: + # 16, 84 (1-sigma) + # 5, 95 (2-sigma) + # 50 (median) + vstat -q $vecname + if ($MIN == $MAX) + $PCT_05 = $MIN + $PCT_16 = $MIN + $PCT_50 = $MIN + $PCT_84 = $MIN + $PCT_95 = $MIN + else + + histogram $vecname _tmpNx $MIN $MAX {($MAX - $MIN) / 1000} -range _tmpdx + cumulative _tmpNx _tmpSx + set _tmpFx = _tmpSx / _tmpSx[-1] + + threshold -saturate -q _tmpdx _tmpFx 0.05; $PCT_05 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.16; $PCT_16 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.50; $PCT_50 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.84; $PCT_84 = $threshold + threshold -saturate -q _tmpdx _tmpFx 0.95; $PCT_95 = $threshold + end + + $dPCT_90 = $PCT_95 - $PCT_05 + $dPCT_68 = $PCT_84 - $PCT_16 +end + +#### run as script +if ($SCRIPT) + if ($argv:n != 4) + echo "USAGE: psvideostats.sh (input) (output) (expname) (chipID)" + exit 2 + end + + psvideostats.imfile $argv:0 $argv:1 $argv:2 $argv:3 + exit 0 +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/queue.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/queue.rawfix.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/queue.rawfix.sh (revision 42531) @@ -1,17 +1,58 @@ #!/bin/csh -if ($#argv != 1) then - echo "USAGE: queue.rawfix.sh (mjddate)" +if ($#argv != 2) then + echo "USAGE: queue.rawfix.sh (list) (filename)" + echo "USAGE: queue.rawfix.sh (night) (YYYYMMDD)" exit 2 endif +set mode = $1 + # make the rawfix database to track rawfix operations -set DBHOST = "ipp060" +set DBHOST = "ippc190" set DBNAME = "rawfix" -set DBUSER = "dvo" -set DBPASS = "dvo" +# set DBUSER = "dvo" +# set DBPASS = "dvo" -## XXX validate the format (YYYYMMDD)? -set mynight = $1 +set DBOPTS = "--defaults-extra-file=~/.my.cnf.rawfix -h $DBHOST $DBNAME" -mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')" +if ($mode == "list") then + set filename = $2 + + # validate the list: could be DATE STATE or just DATE (prcol 1)? + + # just a big list of dates? + foreach night (`cat $filename`) + queue.rawfix.sh night $night + end + exit 0 +endif + +if ($mode == "night") then + ## XXX validate the format (YYYYMMDD)? + set mynight = $2 + + set year = `echo $mynight | awk '{print substr($0, 1, 4)}'` + if (($year < 2005) || ($year > 2030)) then + echo "invalid date $mynight" + exit 2 + endif + + set month = `echo $mynight | awk '{print 1*substr($0, 5, 2)}'` + if (($month < 1) || ($month > 12)) then + echo "invalid date $mynight" + exit 2 + endif + + set day = `echo $mynight | awk '{print 1*substr($0, 7, 2)}'` + if (($day < 1) || ($day > 31)) then + echo "invalid date $mynight" + exit 2 + endif + + mysql $DBOPTS -e "insert into nights (dateobs, state) values ('$mynight', 'new')" + exit 0 +endif + +echo "ERROR: unknown mode $mode" +exit 2 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.advance.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.advance.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.advance.pt (revision 42531) @@ -0,0 +1,54 @@ + +$MAX_ACTIVE_NIGHTS = 10 + +## regularly run the command advance.rawfits.sh +task nights.advance + periods -poll 10 + periods -exec 10 + periods -timeout 30 + host local + active true + + stdout $LOGDIR/advance.log + stderr $LOGDIR/advance.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command advance.rawfix.pl --mode all --topdir $mypath --Nmax $MAX_ACTIVE_NIGHTS + end + + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks + task.exit 0 + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +macro nights.advance.on + task nights.advance + active true + end +end + +macro nights.advance.off + task nights.advance + active false + end +end + + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.burntool.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.burntool.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.burntool.pt (revision 42531) @@ -0,0 +1,292 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +if (not($?RAWFIX_BURNTOOL_INIT)) + book init burntool.book + book init burnhost.book + $RAWFIX_BURNTOOL_INIT = 1 +end + +$MAX_BURNHOST = 2 + +# select nights in state burntool.new +# output: burntool.book +task nights.burntool.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/burntool.load.stdout.log + stderr $LOGDIR/burntool.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + command check.rawfix.sh state burntool.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout burntool.book + book delpage burntool.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: burnhost.book +task nights.burntool.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host local + npending 1 + # note: the tasks which return their results via stdout + # seem to fail if run remotely. + + stdout $LOGDIR/burntool.loadhosts.stdout.log + stderr $LOGDIR/burntool.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages burntool.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in burntool.book (pantaskState == INIT) + book getpage burntool.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword burntool.book $pageName pantaskState RUN + book getword burntool.book $pageName dateobs -var DATEOBS + book getword burntool.book $pageName iter -var ITER + option $pageName + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.burntool.runhosts.v%d" $mypath $DATEOBS $ITER + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + # get_hosts_fixchip.sh selects the most used machine at ITC + # the command returns these fields: dateobs stage fixhost chipID + # NOTE: stage == ITER + command get_hosts_fixchip.sh $mypath $DATEOBS $ITER + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # queueprint stdout + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout burnhost.book + book setword burntool.book $options:0 NHOST {$Nhosts - 1} + book setword burntool.book $options:0 NDONE 0 + # book list + # book listbook burntool.book + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword burntool.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword burntool.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword burntool.book $options:0 pantasksQuality TIMEOUT + end +end + +task nights.burntool.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages burnhost.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in burnhost.book (pantaskState == INIT) + book getpage burnhost.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword burnhost.book $pageName pantaskState RUN + book getword burnhost.book $pageName dateobs -var DATEOBS + book getword burnhost.book $pageName iter -var ITER + book getword burnhost.book $pageName fixhost -var BURNHOST + book getword burnhost.book $pageName chipID -var CHIP_ID + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf STARTDATE "%s/%s/%s" $YEAR $MONTH $DAY + sprintf ENDDATE "%s/%s/%s,23:59:59" $YEAR $MONTH $DAY + + option $pageName $DATEOBS $YEAR + # save DATEOBS so we can find the md5sum.book entry + # also save YEAR for output log + + stdout $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER + stderr $mypath/$YEAR/$DATEOBS/log.burntool.run.$CHIP_ID.v$ITER + + host -required $BURNHOST + ## check if this host needs to be launched + if (not($?host:$BURNHOST)) + echo "host not defined, creating : $BURNHOST" + $host:$BURNHOST = 0 + end + if ($host:$BURNHOST < $MAX_BURNHOST) + control host add $BURNHOST + $host:$BURNHOST ++ + end + + sprintf psvidOutput "%s/%s/%s" $mypath $YEAR $DATEOBS + + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command ipp_apply_burntool_psvideo.pl --dateobs_begin $STARTDATE --dateobs_end $ENDDATE --class_id $CHIP_ID --dbname gpc1 --psvidOutput $psvidOutput + end + + task.exit 0 + nights.burntool.exit OK + end + + # exit values other than 0: + task.exit default + nights.burntool.exit FAIL + showcommand failure + end + + task.exit crash + nights.burntool.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.burntool.exit TIMEOUT + showcommand timeout + end +end + +# output: burnhost.book +task nights.burntool.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/burntool.clean.stdout.log + stderr $LOGDIR/burntool.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages burntool.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in burntool.book (pantaskState == STOP) + book getpage burntool.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage burntool.book $pageName + + # book setword burntool.book $pageName pantaskState RUN + book getword burntool.book $pageName dateobs -var DATEOBS + book getword burntool.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS burntool.done + else + command update.rawfix.sh $DATEOBS burntool.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword burntool.book $options:0 pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword burntool.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword burntool.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword burntool.book $options:0 pantasksQuality TIMEOUT + end +end + +######################### UTILITIES ################################# + +macro nights.burntool.exit + if ($0 != 2) + echo "USAGE: nights.burntool.exit (quality)" + break + end + + book setword burnhost.book $options:0 pantaskState DONE + book getword burntool.book $options:1 NHOST -var Nhosts + book getword burntool.book $options:1 NDONE -var Ndone + book getword burntool.book $options:1 stage -var ITER + $Ndone ++ + book setword burntool.book $options:1 NDONE $Ndone + + sprintf tmpline "%s/%s/%s/log.burntool.runhosts.v%d" $mypath $options:2 $options:1 $ITER + if ("$1" != "OK") + book setword burntool.book $options:1 pantasksQuality $1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword burntool.book $options:1 pantaskState STOP + end + book delpage burnhost.book $options:0 +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.ckchip.pt (revision 42531) @@ -30,8 +30,11 @@ end + # need to delete DONE entries here so the book is not repopulated + # by already-running load tasks task.exit 0 # convert the stdout 'queue' to entries in a book # fields in the table will become words in the page queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout ckchip.book + book delpage ckchip.book -key pantaskState DONE end @@ -52,12 +55,9 @@ task nights.ckchip.run - periods -poll $RPOLL periods -exec $REXEC periods -timeout 60 - npending 1 - - stdout $LOGDIR/ckchip.run.stdout.log - stderr $LOGDIR/ckchip.run.stderr.log + host -required $HOST_CKCHIP + npending 2 task.exec @@ -75,4 +75,5 @@ book setword ckchip.book $pageName pantaskState RUN book getword ckchip.book $pageName dateobs -var DATEOBS + book getword ckchip.book $pageName iter -var ITER option $pageName @@ -81,16 +82,19 @@ substr $DATEOBS 6 2 DAY + # this is the first place in the proces that this location is known + mkdir $mypath/$YEAR/$DATEOBS + stdout $mypath/$YEAR/$DATEOBS/log.ckchip.runhosts.v$ITER + stderr $mypath/$YEAR/$DATEOBS/log.ckchip.runhosts.v$ITER + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - command check_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt + # this command accesses the Nebulous and GPC databases, so it must run at ITC. + command check_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage $ITER end - ## XX delete the page from the book if done? + # mark the entry done, will be deleted above in .load task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword ckchip.book $options:0 pantaskState DONE exec update.rawfix.sh $options:0 ckchip.done - book delpage ckchip.book $options:0 end @@ -116,2 +120,7 @@ end +# option for writing stdout / stderr to a file +# queuepush stdout "------- end of stdout ---------" +# queuesave stdout $options:0/log.ckchip.v$ITER +# queuepush stderr "------- end of stderr ---------" +# queuesave stderr $options:0/log.ckchip.v$ITER Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt (revision 42531) @@ -9,6 +9,9 @@ if (not($?RAWFIX_FIXCHIP_INIT)) book init fixchip.book + book init fixhost.book $RAWFIX_FIXCHIP_INIT = 1 end + +$MAX_FIXHOST = 2 # select nights in state fixchip.new @@ -27,5 +30,4 @@ task.exec - # command check.rawfix.sh state ckchip.done command check.rawfix.sh state fixchip.new end @@ -35,30 +37,34 @@ # fields in the table will become words in the page queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout fixchip.book - end - - # exit values other than 0: - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - -task nights.fixchip.run - + book delpage fixchip.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +# output: fixhost.book +task nights.fixchip.loadhosts periods -poll $RPOLL periods -exec $REXEC periods -timeout 60 + host local npending 1 - - stdout $LOGDIR/fixchip.run.stdout.log - stderr $LOGDIR/fixchip.run.stderr.log + # note: the tasks which return their results via stdout + # seem to fail if run remotely. + + stdout $LOGDIR/fixchip.loadhosts.stdout.log + stderr $LOGDIR/fixchip.loadhosts.stderr.log task.exec @@ -76,5 +82,73 @@ book setword fixchip.book $pageName pantaskState RUN book getword fixchip.book $pageName dateobs -var DATEOBS + book getword fixchip.book $pageName iter -var ITER option $pageName + # save the page so we can use it below + + substr $DATEOBS 0 4 YEAR + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + # the command returns these fields: dateobs stage fixhost chipID + # NOTE: stage == ITER + command get_hosts_fixchip.sh $mypath $DATEOBS $ITER + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # queueprint stdout + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book + book setword fixchip.book $options:0 NHOST {$Nhosts - 1} + book setword fixchip.book $options:0 NDONE 0 + # book list + # book listbook fixchip.book + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword fixchip.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip.book $options:0 pantasksQuality TIMEOUT + end +end + +task nights.fixchip.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixhost.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixhost.book (pantaskState == INIT) + book getpage fixhost.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixhost.book $pageName pantaskState RUN + book getword fixhost.book $pageName dateobs -var DATEOBS + book getword fixhost.book $pageName iter -var ITER + book getword fixhost.book $pageName fixhost -var FIXHOST + book getword fixhost.book $pageName chipID -var CHIP_ID substr $DATEOBS 0 4 YEAR @@ -83,14 +157,92 @@ sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - command fix_chip_locations.pl --dateobs $FULLDATE --chiplist gpc1.chips.txt - end - - ## XX delete the page from the book if done? - task.exit 0 - # convert the stdout 'queue' to entries in a book - # fields in the table will become words in the page - # use the queuesize (Nhosts) to track how many jobs are done / to be done - exec update.rawfix.sh $options:0 fixchip.done - book delpage fixchip.book $options:0 + option $pageName $DATEOBS $YEAR + # save DATEOBS so we can find the md5sum.book entry + # also save YEAR for output log + + stdout $mypath/$YEAR/$DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER + stderr $mypath/$YEAR/$DATEOBS/log.fixchip.run.$CHIP_ID.v$ITER + + host -required $FIXHOST + ## check if this host needs to be launched + if (not($?host:$FIXHOST)) + echo "host not defined, creating : $FIXHOST" + $host:$FIXHOST = 0 + end + if ($host:$FIXHOST < $MAX_FIXHOST) + control host add $FIXHOST + $host:$FIXHOST ++ + end + + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --chipID $CHIP_ID --stage $ITER + end + + task.exit 0 + nights.fixchip.exit OK + end + + # exit values other than 0: + task.exit default + nights.fixchip.exit FAIL + showcommand failure + end + + task.exit crash + nights.fixchip.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.fixchip.exit TIMEOUT + showcommand timeout + end +end + +# output: fixhost.book +task nights.fixchip.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/fixchip.clean.stdout.log + stderr $LOGDIR/fixchip.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in fixchip.book (pantaskState == STOP) + book getpage fixchip.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage fixchip.book $pageName + + # book setword fixchip.book $pageName pantaskState RUN + book getword fixchip.book $pageName dateobs -var DATEOBS + book getword fixchip.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS fixchip.done + else + command update.rawfix.sh $DATEOBS fixchip.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" + book setword fixchip.book $options:0 pantaskState DONE end @@ -99,5 +251,4 @@ showcommand failure book setword fixchip.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 fixchip.fail end @@ -105,5 +256,4 @@ showcommand crash book setword fixchip.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 fixchip.fail end @@ -112,6 +262,35 @@ showcommand timeout book setword fixchip.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 fixchip.fail - end -end - + end +end + +######################### UTILITIES ################################# + +macro nights.fixchip.exit + if ($0 != 2) + echo "USAGE: nights.fixchip.exit (quality)" + break + end + + # options are : pagename DATEOBS YEAR + + book setword fixhost.book $options:0 pantaskState DONE + book getword fixchip.book $options:1 NHOST -var Nhosts + book getword fixchip.book $options:1 NDONE -var Ndone + book getword fixchip.book $options:1 stage -var ITER + $Ndone ++ + book setword fixchip.book $options:1 NDONE $Ndone + + sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $options:2 $options:1 $ITER + if ("$1" != "OK") + book setword fixchip.book $options:1 pantasksQuality $1 + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword fixchip.book $options:1 pantaskState STOP + end + book delpage fixhost.book $options:0 +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5chk.pt (revision 42531) @@ -0,0 +1,120 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the initial chip instance checks +## use the top-level rawfix.pt to load + +# these tasks run the steps: +# -- check_md5s_dateobs.pl --dateobs YYYY/MM/DD + +if (not($?RAWFIX_MD5CHK_INIT)) + book init md5chk.book + $RAWFIX_MD5CHK_INIT = 1 +end + +# select nights in state md5chk.new +# output: md5chk.book +task nights.md5chk.load + host local + periods -poll $TPOLL + periods -exec $TEXEC + periods -timeout 30 + + stdout $LOGDIR/md5chk.load.stdout.log + stderr $LOGDIR/md5chk.load.stderr.log + + # only run one of these tasks at a time + npending 1 + + task.exec + # command check.rawfix.sh state ckchip.done + command check.rawfix.sh state md5chk.new + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5chk.book + book delpage md5chk.book -key pantaskState DONE + end + + # exit values other than 0: + task.exit default + showcommand failure + end + + task.exit crash + showcommand crash + end + + # operation times out? + task.exit timeout + showcommand timeout + end +end + +task nights.md5chk.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages md5chk.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in md5chk.book (pantaskState == INIT) + book getpage md5chk.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword md5chk.book $pageName pantaskState RUN + book getword md5chk.book $pageName dateobs -var DATEOBS + book getword md5chk.book $pageName iter -var ITER + option $pageName + + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 4 2 MONTH + substr $DATEOBS 6 2 DAY + sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY + + # does the output go here? + stdout $mypath/$YEAR/$DATEOBS/log.md5chk.v$ITER + stderr $mypath/$YEAR/$DATEOBS/log.md5chk.v$ITER + + ## this command does NOT use Nebulous or GPC so it can run anywhere + command check_md5s_dateobs.pl --topdir $mypath --dateobs $FULLDATE --stage $ITER + end + + ## XX delete the page from the book if done? + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + # use the queuesize (Nhosts) to track how many jobs are done / to be done + book setword md5chk.book $options:0 pantaskState DONE + exec update.rawfix.sh $options:0 md5chk.done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword md5chk.book $options:0 pantasksQuality FAIL + exec update.rawfix.sh $options:0 md5chk.fail + end + + task.exit crash + showcommand crash + book setword md5chk.book $options:0 pantasksQuality CRASH + exec update.rawfix.sh $options:0 md5chk.fail + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword md5chk.book $options:0 pantasksQuality TIMEOUT + exec update.rawfix.sh $options:0 md5chk.fail + end +end + Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt (revision 42531) @@ -12,4 +12,6 @@ $RAWFIX_MD5SUM_INIT = 1 end + +$MAX_MD5HOST = 2 # select nights which are ready for md5sum calcs @@ -35,4 +37,5 @@ # fields in the table will become words in the page queue2book -raw-columns -key dateobs -uniq -setword pantaskState INIT -setword pantasksQuality OK stdout md5sum.book + book delpage md5sum.book -key pantaskState DONE end @@ -54,9 +57,11 @@ # output: md5host.book task nights.md5sum.loadhosts - periods -poll $RPOLL periods -exec $REXEC periods -timeout 60 + host local npending 1 + # note: the tasks which return their results via stdout + # seem to fail if run remotely. stdout $LOGDIR/md5sum.loadhosts.stdout.log @@ -77,13 +82,18 @@ book setword md5sum.book $pageName pantaskState RUN book getword md5sum.book $pageName dateobs -var DATEOBS + book getword md5sum.book $pageName iter -var ITER option $pageName # save the page so we can use it below + substr $DATEOBS 0 4 YEAR + # write an entry point in the log for this night - sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $DATEOBS + sprintf tmpline "%s/%s/%s/log.md5sum.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER date -var mydate exec echo "# start runhosts $mydate" >> $tmpline - command get_hosts_md5s.sh $DATEOBS + # the command returns these fields: dateobs stage md5host + # NOTE: stage == ITER + command get_hosts_md5s.sh $mypath $DATEOBS $ITER end @@ -91,8 +101,11 @@ # convert the stdout 'queue' to entries in a book # fields in the table will become words in the page + # queueprint stdout queuesize stdout -var Nhosts queue2book -raw-columns -key dateobs:md5host -uniq -setword pantaskState INIT stdout md5host.book book setword md5sum.book $options:0 NHOST {$Nhosts - 1} book setword md5sum.book $options:0 NDONE 0 + # book list + # book listbook md5sum.book # use the queuesize (Nhosts) to track how many jobs are done / to be done end @@ -116,13 +129,11 @@ end +# runs on the hosts identified by get_hosts_md5s.sh # output: md5host.book -task nights.md5sum.runhosts +task nights.md5sum.run periods -poll $RPOLL periods -exec $REXEC periods -timeout 60 - - stdout $LOGDIR/md5sum.runhosts.stdout.log - stderr $LOGDIR/md5sum.runhosts.stderr.log task.exec @@ -138,12 +149,19 @@ if ("$pageName" == "NULL") break + # XXX NOTE: the output field is called 'stage' book setword md5host.book $pageName pantaskState RUN book getword md5host.book $pageName dateobs -var DATEOBS + book getword md5host.book $pageName stage -var ITER book getword md5host.book $pageName md5host -var MD5HOST - option $pageName $DATEOBS + substr $DATEOBS 0 4 YEAR + + option $pageName $DATEOBS $YEAR # save DATEOBS so we can find the md5sum.book entry - - # XXX TEST: $USE_HOST = ipp060 + # also save YEAR for output log + + # does the output go here? + stdout $mypath/$YEAR/$DATEOBS/log.md5sum.$MD5HOST.v$ITER + stderr $mypath/$YEAR/$DATEOBS/log.md5sum.$MD5HOST.v$ITER $USE_HOST = $MD5HOST @@ -154,30 +172,30 @@ $host:$USE_HOST = 0 end - if (not($host:$USE_HOST)) + if ($host:$USE_HOST < $MAX_MD5HOST) control host add $USE_HOST - $host:$USE_HOST = 1 + $host:$USE_HOST ++ end ## even when testing, use the original hostname here: - $input = $mypath/$DATEOBS/$MD5HOST.inst.txt + $input = $mypath/$DATEOBS/$MD5HOST.inst.$ITER.txt # need to make sure this will work on the ipp b-nodes: # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs periods -exec 0.05 - command $mypath/get_md5s_instances.pl --hostname $USE_HOST --input $input + command get_md5s_instances.pl --hostname $USE_HOST --topdir $mypath --dateobs $DATEOBS --stage $ITER end task.exit 0 - nights.mdsum.exit OK + nights.md5sum.exit OK end # exit values other than 0: task.exit default - nights.mdsum.exit FAIL + nights.md5sum.exit FAIL showcommand failure end task.exit crash - nights.mdsum.exit CRASH + nights.md5sum.exit CRASH showcommand crash end @@ -185,5 +203,5 @@ # operation times out? task.exit timeout - nights.mdsum.exit TIMEOUT + nights.md5sum.exit TIMEOUT showcommand timeout end @@ -210,5 +228,5 @@ # look for completed nights in md5sum.book (pantaskState == DONE) - book getpage md5sum.book 0 -var pageName -key pantaskState DONE + book getpage md5sum.book 0 -var pageName -key pantaskState STOP if ("$pageName" == "NULL") break @@ -232,5 +250,5 @@ # now that the entry is set to done in the db, remove the pantasks book entry echo "cleaning this page: $options:0" - book delpage md5sum.book $options:0 + book setword md5sum.book $options:0 pantaskState DONE end @@ -255,18 +273,22 @@ ######################### UTILITIES ################################# -macro nights.mdsum.exit +macro nights.md5sum.exit if ($0 != 2) - echo "USAGE: nights.mdsum.exit (quality)" + echo "USAGE: nights.md5sum.exit (quality)" break end + + # options are : pagename DATEOBS YEAR book setword md5host.book $options:0 pantaskState DONE book getword md5sum.book $options:1 NHOST -var Nhosts book getword md5sum.book $options:1 NDONE -var Ndone + book getword md5sum.book $options:1 stage -var ITER $Ndone ++ book setword md5sum.book $options:1 NDONE $Ndone + + sprintf tmpline "%s/%s/%s/log.md5sum.runhosts.v%d" $mypath $options:2 $options:1 $ITER if ("$1" != "OK") book setword md5sum.book $options:1 pantasksQuality $1 - sprintf tmpline "%s/%s/md5sum.runhosts.log" $mypath $options:1 exec echo "failure $1 $options:0" >> $tmpline else @@ -274,5 +296,5 @@ end if ($Ndone == $Nhosts) - book setword md5sum.book $options:1 pantaskState DONE + book setword md5sum.book $options:1 pantaskState STOP end book delpage md5host.book $options:0 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.pt (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/rawfix.pt (revision 42531) @@ -1,21 +1,3 @@ ## pantasks scripts to manage the raw exposure data repair / cleanup - -$TPOLL = 5 -$TEXEC = 5 -$RPOLL = 5 -$REXEC = 5 -$VERBOSE = 1 -$NETWORK = 1 - -# use this to find the files in this local directory -$mypath = /home/panstarrs/eugene/rawfix.20230221 - -if (not($?LOGDIR)) - $LOGDIR = $mypath/pantasks_logs - mkdir $LOGDIR -end - -input rawfix.ckchip.pt -# input rawfix.md5sum.pt ######################### UTILITIES ################################# Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/select.nights.sql =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/select.nights.sql (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/select.nights.sql (revision 42531) @@ -0,0 +1,1 @@ +select count(datename) as Nexp, datename from (select exp_name, filter, date_format(dateobs,"%Y%m%d") as datename from rawExp where (dateobs >= '2010/05/01') and (dateobs < '2011/01/01')) as tmp group by datename order by Nexp desc Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/task.test.pt =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/task.test.pt (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/task.test.pt (revision 42531) @@ -0,0 +1,59 @@ +## pantasks scripts to manage the raw exposure data repair / cleanup +## this is for the md5sum calculations +## use the top-level rawfix.pt to load + +$RPOLL = 1 +$REXEC = 1 + +# use this to find the files in this local directory +# $mypath = /home/panstarrs/eugene/rawfix.20230221 + +# NOTE: use a directory located on a b-node so the results can +# be seen by both ATRC and ITC machines +$mypath = /data/ippb07.0/home/eugene/rawfix.20230221 + +if (not($?LOGDIR)) + $LOGDIR = $mypath/pantasks_logs + mkdir $LOGDIR +end + +#### the scripts below take the input files (YYYYMMDD/*.inst.txt) +#### and run get_md5s_instances.pl on the hosts associated with those lists +#### the outputs are files (YYYYMMDD/*.md5s.txt) + +if (not($?RAWFIX_MD5SUM_INIT)) + book init md5sum.book + book init md5host.book + $RAWFIX_MD5SUM_INIT = 1 +end + +# output: md5host.book +task nights.md5sum.loadhosts + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + host anyhost + npending 1 + + #stdout test.loadhosts.stdout.log + #stderr test.loadhosts.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec 1000 + + command $mypath/get_hosts_md5s.sh $mypath 20101129 v0 + end + + task.exit 0 + queuesize stdout -var Nhosts + for i 0 $Nhosts + queuepop stdout -var line:$i + end + echo "done with nights.md5sum.loadhosts, $Nhosts" + queueprint stdout + end +end + +## It looks like stdout buffer is not completely filled, and i +## it has an extra line at the start (stdout 1) Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update.rawfix.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update.rawfix.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update.rawfix.sh (revision 42531) @@ -2,5 +2,5 @@ if ($#argv < 1) then - echo "USAGE: update.rawfix.sh (night) (state) [quality]" + echo "USAGE: update.rawfix.sh (night) (state) [iter]" echo " EG: update.rawfix.sh 20100501 new" exit 2 @@ -10,12 +10,18 @@ set state = $2 +set iter = "" +if ($#argv == 3) then + set iter = ", iter = $3" +endif + # make the rawfix database to track rawfix operations -set DBHOST = "ipp060" +set DBHOST = "ippc190" set DBNAME = "rawfix" -set DBUSER = "dvo" -set DBPASS = "dvo" +# set DBUSER = "dvo" +# set DBPASS = "dvo" -set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" +# set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" +set DBOPTS = "--defaults-extra-file=~/.my.cnf.rawfix -B --skip-column-names -h $DBHOST $DBNAME" -mysql $DBOPTS -e "update nights set state = '$state' where (dateobs = '$mynight')" +mysql $DBOPTS -e "update nights set state = '$state' $iter where (dateobs = '$mynight')" exit 0 Index: /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update_chip_stats.pl =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update_chip_stats.pl (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rawfix.20230221/src/update_chip_stats.pl (revision 42531) @@ -0,0 +1,208 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: update_chip_locations.pl --dateobs (night) --stage (0,1,etc) --chiplist (file) +# nebulous.config & chiplist.txt are at topdir + +my ($fulldate, $dateword, $chiplist, $stage, $topdir, $basedir) = &parse_cmdopts; +my ($raw_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +my $Ngood = 0; +my $Ndead = 0; +my $Ngone = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; +my $Nfail = 0; + +my $Nchip = 0; # counts the number of instances + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# loop over all chip_ids & read in stats +foreach my $chip_id (@chip_ids) { + + open (STATFILE, "$basedir/$dateword/$chip_id.stats.$stage.txt"); + my @lines = ; + close (STATFILE); + + my $N = 0; + $Nchip += &parse_line ($lines[$N], "Nchip:"); $N ++; + $Ngood += &parse_line ($lines[$N], "Ngood:"); $N ++; + $Ngone += &parse_line ($lines[$N], "Ngone:"); $N ++; + $Ndead += &parse_line ($lines[$N], "Ndead:"); $N ++; + $Nsing += &parse_line ($lines[$N], "Nsing:"); $N ++; + $Ndups += &parse_line ($lines[$N], "Ndups:"); $N ++; + $Nsbck += &parse_line ($lines[$N], "Nsbck:"); $N ++; + $Nxbck += &parse_line ($lines[$N], "Nxbck:"); $N ++; + $Ncul1 += &parse_line ($lines[$N], "Ncul1:"); $N ++; + $Ncul2 += &parse_line ($lines[$N], "Ncul2:"); $N ++; + $Ncul3 += &parse_line ($lines[$N], "Ncul3:"); $N ++; + $Nfail += &parse_line ($lines[$N], "Nfail:"); $N ++; +} + +print "Nchip: $Nchip\n"; +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; +print "Nfail: $Nfail\n"; + +my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail; +if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } + +# update the stats in the database +my $sql_update_stats = "UPDATE nights set"; +$sql_update_stats .= " Nchip = $Nchip"; +$sql_update_stats .= ", Ngood = $Ngood"; +$sql_update_stats .= ", Ndead = $Ndead"; +$sql_update_stats .= ", Ngone = $Ngone"; +$sql_update_stats .= ", Nsing = $Nsing"; +$sql_update_stats .= ", Ndups = $Ndups"; +$sql_update_stats .= ", Nsbck = $Nsbck"; +$sql_update_stats .= ", Nxbck = $Nxbck"; +$sql_update_stats .= ", Ncul1 = $Ncul1"; +$sql_update_stats .= ", Ncul2 = $Ncul2"; +$sql_update_stats .= ", Ncul3 = $Ncul3"; +$sql_update_stats .= ", Nfail = $Nfail"; +$sql_update_stats .= " WHERE dateobs = $dateword"; + +my $query = $raw_dbh->prepare($sql_update_stats); +$query->execute () or die "invalid response from RAW database\n"; + +exit 0; + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_line { + + my $line = $_[0]; + my $name = $_[1]; + + my @words = split (/\s+/,$line); + + unless ($name eq $words[0]) { die "unexpected entry $line\n"; } + + return $words[1]; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + my ($yearobs) = substr ($dateword, 0, 4); + my $basedir = "$topdir/$yearobs"; + + return ($fulldate, $dateword, $chiplist, $stage, $topdir, $basedir); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /RAW_NAME /) { $rawdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_HOST /) { $rawdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_USER /) { $rawdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PASS /) { $rawdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PORT /) { $rawdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "raw dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name; + pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host; + pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user; + pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port; + pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass; + + my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host"; + if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; } + my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($raw_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} + Index: /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.find.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.find.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.find.sh (revision 42531) @@ -0,0 +1,69 @@ +#!/bin/csh + +set run = v0 +set version = 20230802 + +# set mode to 'find' or 'find.test' +set mode = find + +set srchost = (ipp100 ipp108 ipp112 ipp114) +set srcvol = ( 0 0 1 0) +set tgthost = (ipp161 ipp161 ipp162 ipp162) +set tgtvol = ( 0 1 0 1) + +set mydir = /data/ipphome1.0/home/eugene/rebalance.20230802 +set fraction = 0.90 + +if ($#argv == 0) then + echo "USAGE: launch.find.sh launch" + echo "USAGE: launch.find.sh find (Nentry)" + echo " eg: launch.find.sh find 1" + exit 2 +endif + +if ("$1" == "launch") then + + set i = 1 + while ($i <= $#srchost) + echo ssh -f $srchost[$i] "cd $mydir && ./launch.find.sh find $i" + ssh -f $srchost[$i] "cd $mydir && ./launch.find.sh find $i" +# echo ssh $f ps ux +# ssh $f ps ux +# ssh $f killall mana +# ssh $f killall find + @ i++ + end + exit 0 +endif + +if ("$1" == "find") then + if ($#argv != 2) then + echo "USAGE: launch.find.sh find (Nentry)" + echo " eg: launch.find.sh find 1" + exit 2 + endif + + set N = $2 + + set hn = $srchost[$N] + set hv = $srcvol[$N] + set tn = $tgthost[$N] + set tv = $tgtvol[$N] + + set myhost = `hostname` + if ($hn != $myhost) then + echo "ERROR: need to run find on the source host ($hn vs $myhost)" + exit 3 + endif + + echo "running on $myhost for source $hn.$hv to target $tn.$tv in mode $mode, version $version" + mkdir -p $hn.$hv + echo "./rebalance.mana $mode $hn $hv $tn $tv $version $fraction > & $hn.$hv/log.$mode.$version.$run &" + ./rebalance.mana $mode $hn $hv $tn $tv $version $fraction > & $hn.$hv/log.$mode.$version.$run & + exit 0 +endif + +echo "USAGE: launch.find.sh launch" +echo "USAGE: launch.find.sh find (Nentry)" +echo " eg: launch.find.sh find 1" +exit 2 Index: /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.insert.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.insert.sh (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.insert.sh (revision 42531) @@ -0,0 +1,54 @@ +#!/bin/csh + +set srclist = (ipp100 ipp101 ipp102 ipp104 ipp105 ipp108 ipp112 ipp114 ipp115 ipp117 ipp123 ipp124 ipp125 ipp126 ipp134 ipp136 ipp137 ipp139) +set tgtlist = (ipp153 ipp153 ipp154 ipp154 ipp155 ipp155 ipp156 ipp156 ipp157 ipp157 ipp153 ipp154 ipp155 ipp156 ipp157 ipp154 ipp155 ipp157) + +set mydir = /data/ipphome1.0/home/eugene/rebalance.20220922 + +# manually set these values: +set run = v0 +set version = 20220922 +set mode = insert.check + +if ($#argv < 1) then + echo "USAGE: launch.insert.sh launch" + echo "USAGE: launch.insert.sh (src) (tgt)" + exit 2 +endif + +if ($#argv == 1) then + if ("$1" != "launch") then + echo "USAGE: launch.insert.sh launch" + echo "USAGE: launch.insert.sh (src) (tgt)" + exit 2 + endif + + set i = 1 + while ($i <= $#srclist) + set src = $srclist[$i] + set tgt = $tgtlist[$i] + echo ssh -f $tgt "cd $mydir && ./launch.insert.sh $src $tgt" + ssh -f $tgt "cd $mydir && ./launch.insert.sh $src $tgt" + @ i++ + end + exit 0 +endif + +if ($#argv != 2) then + echo "USAGE: launch.insert.sh launch" + echo "USAGE: launch.insert.sh (src) (tgt)" + exit 2 +endif + +set src = $1 +set tgt = $2 + +set v = 0 ; set h = $src; set t = $tgt; ./rebalance.mana $mode $h $v $t $v $version > & $h.$v/log.$mode.$version.$run & +set v = 1 ; set h = $src; set t = $tgt; ./rebalance.mana $mode $h $v $t $v $version > & $h.$v/log.$mode.$version.$run & + +# e.g.: ssh -f ipp127 "cd `pwd` && ./launch.rsync.sh" + +# echo ssh $f ps ux +# ssh $f ps ux +# ssh $f killall mana +# ssh $f killall find Index: /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.rsync.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.rsync.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/rebalance/launch.rsync.sh (revision 42531) @@ -1,79 +1,59 @@ #!/bin/csh -# set list = "ipp100 ipp101 ipp102 ipp104 ipp105 ipp108 ipp112 ipp114 ipp115 ipp117 ipp123 ipp124 ipp125 ipp126 ipp134 ipp136 ipp137 ipp139" -# set list = "ipp105 ipp108 ipp112 ipp114 ipp115 ipp123 ipp124 ipp125 ipp126 ipp134 ipp136 ipp137 ipp139" - set list = "ipp100 ipp101 ipp102 ipp104 ipp117" -set mydir = /data/ipphome1.0/home/eugene/rebalance.20220922 +set program = `basename $0` -if ($#argv == 1) then - if ("$1" != "launch") then - echo "USAGE: launch.find.sh launch" - exit 2 - endif - foreach f ($list) - echo ssh -f $f "cd $mydir && ./launch.rsync.sh" - ssh -f $f "cd $mydir && ./launch.rsync.sh" -# echo ssh $f ps ux -# ssh $f ps ux -# ssh $f killall mana -# ssh $f killall find +set run = v0 +set version = 20230802 +set mode = rsync + +set srchost = (ipp100 ipp108 ipp112 ipp114) +set srcvol = ( 0 0 1 0) +set tgthost = (ipp161 ipp161 ipp162 ipp162) +set tgtvol = ( 0 1 0 1) + +set mydir = /data/ipphome1.0/home/eugene/rebalance.20230802 + +if ($#argv == 0) then + echo "USAGE: $program launch" + echo "USAGE: $program run (Nentry)" + echo " eg: $program run 1" + exit 2 +endif + +if ("$1" == "launch") then + + set i = 1 + while ($i <= $#srchost) + echo ssh -f $srchost[$i] "cd $mydir && ./$program run $i" + ssh -f $srchost[$i] "cd $mydir && ./$program run $i" + @ i++ end exit 0 endif -set run = v0 -set version = 20220922 +if ("$1" == "run") then + if ($#argv != 2) then + echo "USAGE: $program run (Nentry)" + echo " eg: $program run 1" + exit 2 + endif -set myhost = `hostname` -set Nhost = `hostname | sed s/ipp//` + set N = $2 -set target = NONE -switch ($myhost) - # total of 23TB - case ipp100 - case ipp101 - case ipp123 - set target = ipp153 - breaksw; + set hn = $srchost[$N] + set hv = $srcvol[$N] + set tn = $tgthost[$N] + set tv = $tgtvol[$N] - # total of 35TB - case ipp102 - case ipp104 - case ipp124 - case ipp136 - set target = ipp154 - breaksw; + set myhost = `hostname` + if ($hn != $myhost) then + echo "ERROR: need to run $mode on the source host ($hn vs $myhost)" + exit 3 + endif - # total of 35TB - case ipp105 - case ipp108 - case ipp125 - case ipp137 - set target = ipp155 - breaksw; - - # total of 23TB - case ipp112 - case ipp114 - case ipp126 - set target = ipp156 - breaksw; - - # total of 38TB - case ipp115 - case ipp117 - case ipp134 - case ipp139 - set target = ipp157 - breaksw; -endsw - -if ("$target" == "NONE") then - echo "need target" - exit 2 + echo "running on $myhost for source $hn.$hv to target $tn.$tv in mode $mode, version $version" + mkdir -p $hn.$hv + echo "./rebalance.mana $mode $hn $hv $tn $tv $version > & $hn.$hv/log.$mode.$version.$run &" + ./rebalance.mana $mode $hn $hv $tn $tv $version > & $hn.$hv/log.$mode.$version.$run & + exit 0 endif - -set v = 0 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync $h $v $t $v $version > & $h.$v/log.rsync.$version.$run & -set v = 1 ; set h = `hostname` ; set t = $target; ./rebalance.mana rsync $h $v $t $v $version > & $h.$v/log.rsync.$version.$run & - -# e.g.: ssh -f ipp127 "cd `pwd` && ./launch.rsync.sh" Index: /branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.css =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.css (revision 42531) +++ /branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.css (revision 42531) @@ -0,0 +1,71 @@ +/* internal link line: c0d0d0 : d8d0f0 + external link line: 303070 + border: 0080c0 + */ + +body { + background-color: #a0d0e0; + padding: 5px; + margin: 5px +} + +a:link, a:visited, a:active { + text-decoration: underline; + font-weight: bold; + color: #ff0000 +} + +a.button { + text-decoration: none; + font-weight: bold; + color: #000000; + background: #0080c0 +} +table.pics { + text-align: left; + background: #0080c0; + font-size: small; + color: #ffffff; + padding: 2px; +} +table.stats { + text-align: left; + background: #ffffff; + font-size: big; + color: #000000; + padding: 10px; +} +table.index { + border: 3px solid #0080c0; + text-align: left; + background: #ffffff; + font-size: big; + color: #000000; + padding: 10px; +} +td.index { + text-align: right; + font-size: normal; + font-weight: normal; + vertical-align: top; + background: #d0d0ff; + background-color: #d0d0ff; + border: 3px solid #0080c0; + padding: 5px; +} +tr.index { + text-align: left; + font-size: small; + font-weight: normal; + vertical-align: top; + background: #d0d0ff; + background-color: #d0d0ff; + border: 3px solid #0080c0; + padding: 5px; +} +td.label { + font-size: large; + font-weight: bold; + padding: 5px; + transform: rotate(90deg); +} Index: /branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.mana.sh =================================================================== --- /branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.mana.sh (revision 42530) +++ /branches/eam_branches/ipp-20230313/tools/eam/skycoverage/coverage.mana.sh (revision 42531) @@ -319,4 +319,65 @@ cplot -pt 0 -sz 0.4 $RCOLOR el eb end + + $RCOLOR = -c grey70 + DR1.boundary +end + +macro DR1.boundary + if (1) + if (0) + data EuclidDR1_polygon.csv + read -skip 1 -csv Re 1 De 2 + cplot -pt 0 -sz 0.5 -c green Re De + end + + # north boundary: + # corner 105, 80 -> 94.7, 56.9 in ecliptic + create Rn 105.2 285 0.1 + set Dn = zero(Rn) + 80 + cplot $RCOLOR -pt box -sz 0.4 Rn Dn + + # west boundary + create elat 50 56.9 0.1 + set elon = zero(elat) + 95 + csystem E C elon elat + cplot $RCOLOR -pt box -sz 0.4 elon elat + + # south boundary west + create elon 95 150 0.1 + set elat = zero(elon) + 50 + csystem E C elon elat + cplot $RCOLOR -pt box -sz 0.4 elon elat + + # south boundary east + create elat 50 80 0.1 + set elon = zero(elat) + 150 + csystem E C elon elat + cplot $RCOLOR -pt box -sz 0.4 elon elat + + # NEP circle + create elon3 150 296 0.1 + set elat3 = zero(elon3) + 80 + csystem E C elon3 elat3 + cplot $RCOLOR elon3 elat3 -pt box -sz 0.4 + + # galactic plane boundary east + create glon 86.5 103 0.1 + set glat = zero(glon) + 25 + csystem G C glon glat + cplot $RCOLOR -pt box -sz 0.4 glon glat + + # east boundary + create Dn 72 80 0.1 + set Rn = zero(Dn) + 285 + csystem E C elon elat + cplot $RCOLOR -pt box -sz 0.4 Rn Dn + + # DR2 south boundary + create elon 100.3 189.5 0.1 + set elat = zero(elon) + 37 + csystem E C elon elat + cplot $RCOLOR -pt box -sz 0.4 elon elat + end end