Index: /trunk/tools/eam/euclid.sample.20210819/coverage.css =================================================================== --- /trunk/tools/eam/euclid.sample.20210819/coverage.css (revision 42399) +++ /trunk/tools/eam/euclid.sample.20210819/coverage.css (revision 42399) @@ -0,0 +1,71 @@ +/* internal link line: c0d0d0 : d8d0f0 + external link line: 303070 + border: 0080c0 + */ + +body { + background-color: #a0d0e0; + padding: 5px; + margin: 5px +} + +a:link, a:visited, a:active { + text-decoration: underline; + font-weight: bold; + color: #ff0000 +} + +a.button { + text-decoration: none; + font-weight: bold; + color: #000000; + background: #0080c0 +} +table.pics { + text-align: left; + background: #0080c0; + font-size: small; + color: #ffffff; + padding: 2px; +} +table.stats { + text-align: left; + background: #ffffff; + font-size: big; + color: #000000; + padding: 10px; +} +table.index { + border: 3px solid #0080c0; + text-align: left; + background: #ffffff; + font-size: big; + color: #000000; + padding: 10px; +} +td.index { + text-align: right; + font-size: normal; + font-weight: normal; + vertical-align: top; + background: #d0d0ff; + background-color: #d0d0ff; + border: 3px solid #0080c0; + padding: 15px; +} +tr.index { + text-align: left; + font-size: small; + font-weight: normal; + vertical-align: top; + background: #d0d0ff; + background-color: #d0d0ff; + border: 3px solid #0080c0; + padding: 15px; +} +td.label { + font-size: large; + font-weight: bold; + padding: 15px; + transform: rotate(90deg); +} Index: /trunk/tools/eam/euclid.sample.20210819/mkstacks =================================================================== --- /trunk/tools/eam/euclid.sample.20210819/mkstacks (revision 42399) +++ /trunk/tools/eam/euclid.sample.20210819/mkstacks (revision 42399) @@ -0,0 +1,84 @@ +#!/bin/tcsh + +if ($#argv != 2) then + echo "USAGE: mkstacks (filter) (test/commit)" + echo " filter: i.00000 or z.00000" + exit 2 +endif + +# the skycell list is generated within dvo: +# +# dvo: catdir /local/ipp/gpc1/tess/CFIS.V0/ +# dvo: skyregion {8*15} {9*15} 38 40 +# dvo: output skycells.txt +# dvo: imlist +# dvo: output stdout + +set filter = "$1" + +set Nlimit = 2 +set Ncells = 2 +if ("$2" == "commit") then + set Nlimit = 1000000 + set Ncells = 1000000 +endif + +set level1 = "stacks.$filter" +set wdirs1 = "webpages/$level1" +set pages1 = "$wdirs1/index.html" +mkdir -p $wdirs1 +echo $pages1 + +# level 1 content: +echo "" > $pages1 + +# remove the "head" bit when ready: +foreach skycell (`cat skycells.txt | awk '{print $2}' | head -n $Ncells`) + + # echo $skycell + set level2 = $level1/$skycell + set wdirs2 = "webpages/$level2" + set pages2 = "$wdirs2/index.html" + mkdir -p $wdirs2 + echo $pages2 + + # level 1 content: + echo " $skycell
" >> $pages1 + + # level 2 content: + echo "" > $pages2 + + foreach path_base (`stacktool -dbname gpc1 -sumskyfile -label UNIONS.DR2.20190430 -skycell_id $skycell -filter $filter -limit $Nlimit | grep path_base | awk '{print $3}'`) + + # echo $path_base + set outbase = `basename $path_base` + + set level3 = $level2/$outbase + set wdirs3 = "webpages/$level3" + set pages3 = "$wdirs3/index.html" + mkdir -p $wdirs3 + echo $pages3 + + # level 2 content: + echo " $outbase
" >> $pages2 + + # level 3 content: + echo "" > $pages3 + + foreach ext (cmf unconv.fits unconv.mask.fits unconv.wt.fits) + set nebname = $path_base.$ext + set nebbase = `basename $nebname` + set realname = `neb-locate -p $nebname` + if ("$realname" == "") then + echo "skipping $nebname" + continue + endif + ln -s $realname $wdirs3/$nebbase + echo " $nebbase
" >> $pages3 + end + echo "" >> $pages3 + end + echo "" >> $pages2 +end + +echo "" >> $pages1 Index: /trunk/tools/eam/euclid.sample.20210819/mkwarps =================================================================== --- /trunk/tools/eam/euclid.sample.20210819/mkwarps (revision 42399) +++ /trunk/tools/eam/euclid.sample.20210819/mkwarps (revision 42399) @@ -0,0 +1,80 @@ +#!/bin/tcsh + +if ($#argv != 2) then + echo "USAGE: mkwarps (filter) (test/commit)" + echo " filter: i.00000 or z.00000" + exit 2 +endif + +# the skycell list is generated within dvo: +# +# dvo: catdir /local/ipp/gpc1/tess/CFIS.V0/ +# dvo: skyregion {8*15} {9*15} 38 40 +# dvo: output skycells.txt +# dvo: imlist +# dvo: output stdout + +set filter = "$1" + +set Nlimit = 2 +set Ncells = 2 +if ("$2" == "commit") then + set Nlimit = 1000000 + set Ncells = 1000000 +endif + +set level1 = "warps.$filter" +set wdirs1 = "webpages/$level1" +set pages1 = "$wdirs1/index.html" +mkdir -p $wdirs1 +echo $pages1 + +# level 1 content: +echo "" > $pages1 + +# remove the "head" bit when ready: +foreach skycell (`cat skycells.txt | awk '{print $2}' | head -n $Ncells`) + + # echo $skycell + set level2 = $level1/$skycell + set wdirs2 = "webpages/$level2" + set pages2 = "$wdirs2/index.html" + mkdir -p $wdirs2 + echo $pages2 + + # level 1 content: + echo " $skycell
" >> $pages1 + + # level 2 content: + echo "" > $pages2 + + foreach path_base (`warptool -dbname gpc1 -warped -label UNIONS.DR2.20190430 -skycell_id $skycell -filter $filter -limit $Nlimit | grep path_base | awk '{print $3}'`) + + # echo $path_base + set outbase = `basename $path_base` + + set level3 = $level2/$outbase + set wdirs3 = "webpages/$level3" + set pages3 = "$wdirs3/index.html" + mkdir -p $wdirs3 + echo $pages3 + + # level 2 content: + echo " $outbase
" >> $pages2 + + # level 3 content: + echo "" > $pages3 + + foreach ext (cmf fits mask.fits wt.fits psf) + set nebname = $path_base.$ext + set nebbase = `basename $nebname` + set realname = `neb-locate -p $nebname` + ln -s $realname $wdirs3/$nebbase + echo " $nebbase
" >> $pages3 + end + echo "" >> $pages3 + end + echo "" >> $pages2 +end + +echo "" >> $pages1 Index: /trunk/tools/eam/euclid.sample.20230206/coverage.css =================================================================== --- /trunk/tools/eam/euclid.sample.20230206/coverage.css (revision 42399) +++ /trunk/tools/eam/euclid.sample.20230206/coverage.css (revision 42399) @@ -0,0 +1,71 @@ +/* internal link line: c0d0d0 : d8d0f0 + external link line: 303070 + border: 0080c0 + */ + +body { + background-color: #a0d0e0; + padding: 5px; + margin: 5px +} + +a:link, a:visited, a:active { + text-decoration: underline; + font-weight: bold; + color: #ff0000 +} + +a.button { + text-decoration: none; + font-weight: bold; + color: #000000; + background: #0080c0 +} +table.pics { + text-align: left; + background: #0080c0; + font-size: small; + color: #ffffff; + padding: 2px; +} +table.stats { + text-align: left; + background: #ffffff; + font-size: big; + color: #000000; + padding: 10px; +} +table.index { + border: 3px solid #0080c0; + text-align: left; + background: #ffffff; + font-size: big; + color: #000000; + padding: 10px; +} +td.index { + text-align: right; + font-size: normal; + font-weight: normal; + vertical-align: top; + background: #d0d0ff; + background-color: #d0d0ff; + border: 3px solid #0080c0; + padding: 15px; +} +tr.index { + text-align: left; + font-size: small; + font-weight: normal; + vertical-align: top; + background: #d0d0ff; + background-color: #d0d0ff; + border: 3px solid #0080c0; + padding: 15px; +} +td.label { + font-size: large; + font-weight: bold; + padding: 15px; + transform: rotate(90deg); +} Index: /trunk/tools/eam/euclid.sample.20230206/get.warp.paths =================================================================== --- /trunk/tools/eam/euclid.sample.20230206/get.warp.paths (revision 42399) +++ /trunk/tools/eam/euclid.sample.20230206/get.warp.paths (revision 42399) @@ -0,0 +1,27 @@ +#!/bin/csh + +if ($#argv != 3) then + echo "USAGE: get.warp.paths (dbname) (skycell_id) (filter)" + exit 2 +endif + +set dbname = $1 +set skycell = $2 +set filter = $3 + +#set query = "$query select count(exp_name), exp_name, exp_id, filter, warp_id, skycell_id, warpSkyfile.data_state, warpRun.state, warpRun.label" + +set query = "" +set query = "$query SELECT count(exp_name), exp_name, filter, warpRun.label, warpSkyfile.path_base" +set query = "$query FROM rawExp join chipRun using (exp_id)" +set query = "$query JOIN camRun using (chip_id)" +set query = "$query JOIN fakeRun using (cam_id)" +set query = "$query JOIN warpRun using (fake_id)" +set query = "$query JOIN warpSkyfile using (warp_id)" +set query = "$query WHERE skycell_id = '$skycell'" +set query = "$query AND warpRun.state = 'full'" +set query = "$query AND filter = '$filter'" +set query = "$query GROUP by exp_name" + +mysql -h ippdb05 -u ippuser -pippuser gpc1 -e "$query" + Index: /trunk/tools/eam/euclid.sample.20230206/mkstacks =================================================================== --- /trunk/tools/eam/euclid.sample.20230206/mkstacks (revision 42399) +++ /trunk/tools/eam/euclid.sample.20230206/mkstacks (revision 42399) @@ -0,0 +1,84 @@ +#!/bin/tcsh + +if ($#argv != 2) then + echo "USAGE: mkstacks (filter) (test/commit)" + echo " filter: i.00000 or z.00000" + exit 2 +endif + +# the skycell list is generated within dvo: +# +# dvo: catdir /local/ipp/gpc1/tess/CFIS.V0/ +# dvo: skyregion {8*15} {9*15} 38 40 +# dvo: output skycells.txt +# dvo: imlist +# dvo: output stdout + +set filter = "$1" + +set Nlimit = 2 +set Ncells = 2 +if ("$2" == "commit") then + set Nlimit = 1000000 + set Ncells = 1000000 +endif + +set level1 = "stacks.$filter" +set wdirs1 = "webpages/$level1" +set pages1 = "$wdirs1/index.html" +mkdir -p $wdirs1 +echo $pages1 + +# level 1 content: +echo "" > $pages1 + +# remove the "head" bit when ready: +foreach skycell (`cat skycells.txt | awk '{print $2}' | head -n $Ncells`) + + # echo $skycell + set level2 = $level1/$skycell + set wdirs2 = "webpages/$level2" + set pages2 = "$wdirs2/index.html" + mkdir -p $wdirs2 + echo $pages2 + + # level 1 content: + echo " $skycell
" >> $pages1 + + # level 2 content: + echo "" > $pages2 + + foreach path_base (`stacktool -dbname gpc1 -sumskyfile -label UNIONS.DR2.20190430 -skycell_id $skycell -filter $filter -limit $Nlimit | grep path_base | awk '{print $3}'`) + + # echo $path_base + set outbase = `basename $path_base` + + set level3 = $level2/$outbase + set wdirs3 = "webpages/$level3" + set pages3 = "$wdirs3/index.html" + mkdir -p $wdirs3 + echo $pages3 + + # level 2 content: + echo " $outbase
" >> $pages2 + + # level 3 content: + echo "" > $pages3 + + foreach ext (cmf unconv.fits unconv.mask.fits unconv.wt.fits) + set nebname = $path_base.$ext + set nebbase = `basename $nebname` + set realname = `neb-locate -p $nebname` + if ("$realname" == "") then + echo "skipping $nebname" + continue + endif + ln -s $realname $wdirs3/$nebbase + echo " $nebbase
" >> $pages3 + end + echo "" >> $pages3 + end + echo "" >> $pages2 +end + +echo "" >> $pages1 Index: /trunk/tools/eam/euclid.sample.20230206/mkwarps =================================================================== --- /trunk/tools/eam/euclid.sample.20230206/mkwarps (revision 42399) +++ /trunk/tools/eam/euclid.sample.20230206/mkwarps (revision 42399) @@ -0,0 +1,125 @@ +#!/bin/tcsh + +if ($#argv != 3) then + echo "USAGE: mkwarps (filter) (pages/paths) (test/commit)" + echo " filter: i.00000 or z.00000" + exit 2 +endif + +# the skycell list is generated within dvo: +# +# dvo: catdir /local/ipp/gpc1/tess/CFIS.V0/ +# dvo: skyregion {8*15} {9*15} 38 40 +# dvo: output skycells.txt +# dvo: imlist +# dvo: output stdout + +set skycellList = "skycells.nep.txt" + +set filter = "$1" + +if (("$2" != "pages") && ("$2" != "paths") && ("$2" != "fix")) then + echo "select either fix, pages or paths" + exit 2 +endif + +set Ncells = 2 +if ("$3" == "commit") then + set Ncells = 1000000 +endif + +## generate the *.paths files for each skycell. +if (("$2" == "paths") || ("$2" == "fix")) then + set level1 = "warps.$filter" + set wdirs1 = "skycells/$level1" + mkdir -p $wdirs1 + + # remove the "head" bit when ready: + foreach skycell (`cat $skycellList | awk '{print $2}' | head -n $Ncells`) + echo $skycell + + set projcell = `echo $skycell | awk -F. '{printf "proj.%s\n", $2}'` + set level2 = $level1/$projcell + set wdirs2 = "skycells/$level2" + mkdir -p $wdirs2 + + # skip if the output file exists? + if ("$2" == "paths") then + if (-e $wdirs2/$skycell.paths) continue + get.warp.paths gpc1 $skycell $filter > $wdirs2/$skycell.paths + else + echo mv $skycell.paths $wdirs2/$skycell.paths + endif + + end + exit 0 +endif + +## generate the web pages +set level1 = "warps.$filter" +set wdirs1 = "webpages/$level1" +set pages1 = "$wdirs1/index.html" +mkdir -p $wdirs1 +echo $pages1 + +# level 1 content: +echo "" > $pages1 + +# remove the "head" bit when ready: +foreach skycell (`cat $skycellList | awk '{print $2}' | head -n $Ncells`) + + # echo $skycell + set projcell = `echo $skycell | awk -F. '{printf "proj.%s\n", $2}'` + set sdirs2 = "skycells/$level1/$projcell" + + set level2 = $level1/$projcell/$skycell + set wdirs2 = "webpages/$level2" + set pages2 = "$wdirs2/index.html" + + mkdir -p $wdirs2 + echo $pages2 + + # level 1 content: + echo " $skycell
" >> $pages1 + + # level 2 content: + echo "" > $pages2 + + foreach path_base (`awk '(NR > 1){print $5}' $sdirs2/$skycell.paths`) + + # echo $path_base + set outbase = `basename $path_base` + + set level3 = $level2/$outbase + set wdirs3 = "webpages/$level3" + set pages3 = "$wdirs3/index.html" + mkdir -p $wdirs3 + echo $pages3 + + # level 2 content: + echo " $outbase
" >> $pages2 + + # level 3 content: + echo "" > $pages3 + + foreach ext (cmf fits mask.fits wt.fits psf) + set nebname = $path_base.$ext + set nebbase = `basename $nebname` + if (-e $wdirs3/$nebbase) then + echo " $nebbase
" >> $pages3 + continue + endif + set realname = `neb-locate -p $nebname` + if ("$realname" == "") then + echo "skipping $nebname" + continue + endif + ln -s $realname $wdirs3/$nebbase + echo " $nebbase
" >> $pages3 + end + echo "" >> $pages3 + end + echo "" >> $pages2 +end + +echo "" >> $pages1 Index: /trunk/tools/eam/euclid.sample.20230206/notes.txt =================================================================== --- /trunk/tools/eam/euclid.sample.20230206/notes.txt (revision 42399) +++ /trunk/tools/eam/euclid.sample.20230206/notes.txt (revision 42399) @@ -0,0 +1,4 @@ +select exp_name, exp_id, filter, warp_id, skycell_id, warpSkyfile.data_state, warpRun.state from rawExp join chipRun using (exp_id) join camRun using (chip_id) join fakeRun using (cam_id) join warpRun using (fake_id) join warpSkyfile using (warp_id) where skycell_id = 'skycell.2528.062' and warpRun.state = 'full' limit 10; + +select count(exp_name), exp_id, filter, warp_id, skycell_id, warpSkyfile.data_state, warpRun.state from rawExp join chipRun using (exp_id) join camRun using (chip_id) join fakeRun using (cam_id) join warpRun using (fake_id) join warpSkyfile using (warp_id) where skycell_id = 'skycell.2528.062' and warpRun.state = 'full' group by filter limit 10 + Index: /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh (revision 42399) @@ -0,0 +1,39 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: check.rawfix.sh (mode) [options]" + echo "USAGE: check.rawfix.sh night (YYYYMMDD)" + echo "USAGE: check.rawfix.sh new" + exit 2 +endif + +set mode = $1 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" + +if ($mode == "night") then + if ($#argv != 2) then + echo "USAGE: check.rawfix.sh night (YYYYMMDD)" + exit 2 + endif + + set mynight = $2 + mysql $DBOPTS -e "select dateobs, state from nights where (dateobs = '$mynight')" + exit 0 +endif + +if ($mode == "new") then + mysql $DBOPTS -e "select dateobs from nights where (state = 'new')" + exit 0 +endif + +echo "ERROR: unknown mode $mode" +exit 2 + + Index: /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.pl (revision 42399) @@ -0,0 +1,385 @@ +#! /usr/bin/env perl + +# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases +my $DEBUG = 3; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: check_chip_locations.pl --dateobs (night) --chiplist (file) +# USAGE: check_chip_locations.pl --dateobs 2010/05/01 --chiplist gpc1.chips.txt + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; +my ($neb_dbh, $gpc_dbh) = &parse_db_config ('nebulous.config'); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# a global to store the number of instances per host (for each chip) +my %hostcount; +my %hostfile; + +# put output files in a directory defined by the night +mkdir $dateword; + +# output files: +# instance lists by host: YYYYMMDD/HOSTNAME.inst.txt +# chip states by chip: YYYYMMDD/CHIPNAME.uris.txt + +foreach my $chip_id (@chip_ids) { + + my $outfile = "$dateword/$chip_id.uris.txt"; + my $outfh; + open ($outfh, ">$outfile"); + + # get the list of chip neb paths for one chip + my @neb_paths = &get_chip_paths ($fulldate, $chip_id); + + # reset the hostcount for this chip + %hostcount = (); + + # find the neb instances for each chip nebulous entry + foreach my $nebname (@neb_paths) { + + # get info for all instances of this chip + # returns a reference to the array of instance data + # also strips neb://hostname.volnum/ from nebname and returns as ext_id + my ($inforows, $ext_id) = &get_chip_instances ($nebname); + + # determine the status of the instances + my ($Nmain, $Nback, $Ndead) = &check_instance_xattr ($inforows, $ext_id); + + &check_instance_state ($outfh, $ext_id, $Nmain, $Nback, $Ndead); + } + + close ($outfh); + + # now save the host count for this chip + my $hostfh; + open ($hostfh, ">$dateword/$chip_id.hosts.txt"); + + # save the host count for this chip + my @hostnames = keys %hostcount; + foreach my $myname (@hostnames) { + print $hostfh "host $myname = $hostcount{$myname}\n"; + } + close ($hostfh); +} + +# save the host count for this chip +my @hostnames = keys %hostfile; +foreach my $myname (@hostnames) { + close ($hostfile{$myname}); + if ($DEBUG > 4) { print "close host instance file for $myname\n"; } +} + +# run neb-replicate for SING entries? +# run neb-replicate for SBCK entries? (does this put a second copy at ATRC or ITC?) +# save the list of instances on each host +## run md5sum on the instances for each host +## only note inconsistent md5sums? (need to save the gpc1 md5sum values) +# rerun to generate a list of the host with max instances per chip? + +### when to cull entries? +## neb-replicate puts the new entry at ATRC by default +## neb-replicate --volume VOL puts the new entry at the specified location +### in order to use this, I need to choose the target volume + +die "done"; + +sub vsystem { + # print STDERR "@_\n"; + return 0; + #my $status = system ("@_"); + #return $status; +} + + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +# get the list of chip neb paths for one chip +sub get_chip_paths { + my $fulldate = $_[0]; + my $chip_id = $_[1]; + + my @nebarray; + + my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$fulldate,00:00:00' and dateobs < '$fulldate,23:59:59'"; + if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } + + my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows chip images\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only uri selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@nebarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @nebarray; +} + +sub get_chip_instances { + + my $nebname = $_[0]; + + # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + + # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + if ($DEBUG > 5) { print "nebname: $ext_id\n"; } + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + + return ($rowref, $ext_id); +} + +# count the number of instances at ITC (0), ATRC (1), or other locations (>1) +# record the instances in a file for each host +# count the number of instances on each host +sub check_instance_xattr { + + my $ext_id = $_[1]; + + my $rowref = $_[0]; + my @inforows = @$rowref; + my $Nrows = @inforows; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + my $Ndead = 0; + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $inforows[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + # the number of returned values per row must match the query above (in get_chip_instances) + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $ext_id\n"; + next; + } + + my $instance = $values[0]; + my $hostname = $values[1]; + my $xattr = $values[3]; + + # count the number of each host by incrementing the hash element corresponding to the hostname: + # NOTE: hostcount is global + $hostcount{$hostname} ++; + + if (not defined $hostfile{$hostname}) { + open ($hostfile{$hostname}, ">$dateword/$hostname.inst.txt"); + ## XXX deal with open failure here + } + my $fh = $hostfile{$hostname}; + print $fh "$instance\n"; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + if ($xattr > 1) { $Ndead ++; } + } + + return ($Nmain, $Nback, $Ndead); +} + +sub check_instance_state { + + my $outfh = $_[0]; + my $extID = $_[1]; + my $Nmain = $_[2]; + my $Nback = $_[3]; + my $Ndead = $_[4]; + + # SKIP : already fixed + if (($Nmain == 1) && ($Nback == 1)) { + print $outfh "GOOD: $extID\n"; + return; + } + + # single copy @ ITC, no backup + # make a backup + if (($Nmain == 1) && ($Nback == 0)) { + print $outfh "SING: $extID\n"; + return; + } + + # ITC files with a single backup copies need to be culled + # cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 1)) { + print $outfh "CULL: $extID\n"; + return; + } + + # multiple copies @ ITC + # make a backup and cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 0)) { + print $outfh "DUPS: $extID\n"; + return; + } + + # no more valid instances + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print $outfh "DEAD: $extID\n"; + return; + } + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print $outfh "GONE: $extID\n"; + return; + } + + # single copy @ ATRC + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print $outfh "SBCK: $extID\n"; + return; + } + + # multiple copies @ ATRC, but none at ITC + # make a backup at ITC, cull ATRC extras + if (($Nmain == 0) && ($Nback > 1)) { + print $outfh "XBCK: $extID\n"; + return; + } + + # unexpected state: + print $outfh "OOPS: $extID\n"; + return; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + my $fulldate = "$date_year/$date_month/$date_day"; + + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist) +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; + pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; + pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; + pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; + pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; + if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } + my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh, $gpc_dbh); +} + + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/check_chip_locations.v0.pl (revision 42399) @@ -0,0 +1,423 @@ +#! /usr/bin/env perl + +# NOTE: in this script, we are using database connections to both NEBULOUS and GPC1 databases + +# TESTRUN can be 1, 2, 3, 4 to get further in the tests +my $TESTRUN = 2; +my $DEBUG = 3; + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($tgthost,$tgtvol,$subdir,$movelist); + +# USAGE: check_chip_locations.pl --dateobs (night) --chip_id (chip_id) +# USAGE: check_chip_locations.pl --dateobs 2010.05.01 --chip_id XY33 + +my ($dateobs, $chip_id); +GetOptions( 'dateobs=s' => \$dateobs, 'chip_id=s' => \$chip_id) || pod2usage(2); + +my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); +my ( $gpcdb_name, $gpcdb_host, $gpcdb_port, $gpcdb_pass, $gpcdb_user ); +my $conffile = 'nebulous.config'; + +# parse the nebulous.config file: +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + + if ($_ =~ /GPC1_NAME /) { $gpcdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_HOST /) { $gpcdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_USER /) { $gpcdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PASS /) { $gpcdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /GPC1_PORT /) { $gpcdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + print STDERR "gpc1 dbh config: $gpcdb_name $gpcdb_host $gpcdb_port $gpcdb_user $gpcdb_pass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; +pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; +pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; +pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; +pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + +pod2usage( -msg => "Cannot configure gpc1 database (dbname)", -exitval => 2) unless defined $gpcdb_name; +pod2usage( -msg => "Cannot configure gpc1 database (dbhost)", -exitval => 2) unless defined $gpcdb_host; +pod2usage( -msg => "Cannot configure gpc1 database (dbuser)", -exitval => 2) unless defined $gpcdb_user; +pod2usage( -msg => "Cannot configure gpc1 database (dbport)", -exitval => 2) unless defined $gpcdb_port; +pod2usage( -msg => "Cannot configure gpc1 database (dbpass)", -exitval => 2) unless defined $gpcdb_pass; + +pod2usage( -msg => "Cannot determine target date", -exitval => 2) unless defined $dateobs; +pod2usage( -msg => "Cannot determine target chip", -exitval => 2) unless defined $chip_id; + +# check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + +my ($date_year, $date_month, $date_day) = split "/",$dateobs; +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); +pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + +my $time = time(); + +my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; +if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } +my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +my $gpcdb_dsn = "DBI:mysql:database=$gpcdb_name:host=$gpcdb_host"; +if ($gpcdb_port ne "NONE") { $gpcdb_dsn .= ":port=$gpcdb_port"; } +my $gpc_dbh = DBI->connect($gpcdb_dsn, $gpcdb_user, $gpcdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# get the list of chip neb paths: +my @nebarray; +if (1) { + my $sql_chip_paths = "SELECT uri from rawImfile WHERE class_id = '$chip_id' and dateobs > '$date_year/$date_month/$date_day,00:00:00' and dateobs < '$date_year/$date_month/$date_day,23:59:59'"; + if ($DEBUG > 5) { print "sql: $sql_chip_paths\n"; } + + my $rowref = $gpc_dbh->selectall_arrayref( $sql_chip_paths ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows chip images\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only uri selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@nebarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } +} + +# find the neb instances for each chip nebulous entry + +my %hostcount; + +foreach my $nebname (@nebarray) { + + # nebname is of the form neb://ipp029.0/gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # but ext_id is of the form gpc1/20100502/o5318g0493q/o5318g0493q.ota33.fits + # my ($ext_id) = $nebname =~ m|neb://\w*/(\S*)|; ---> what if we do not have the host.volnum portion? (this attempt fails) + my ($ext_id) = $nebname =~ m|neb://[0-9a-zA-Z]*\.[0-9]/(\S*)|; + if ($DEBUG > 5) { print "nebname: $ext_id\n"; } + + my $sql_chip_instances = "SELECT uri, host, available, xattr, cab_id from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id'"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 5) { print "found $Nrows instances\n"; } + + if (0) { + foreach my $valref (@tmparray) { + my @values = @$valref; + # number of values should match list above (can this even fail?) + my $Nvalues = @values; + if ($Nvalues != 5) { print "ERROR: wrong number of fields?\n"; next; } + + if ($DEBUG > 2) { print "uri: $values[0], host: $values[1], avail: $values[2], xattr: $values[3], cab: $values[4]\n"; } + } + } + + # my $summary = "$ext_id $so_id"; + my $summary = "$ext_id"; + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + my $Ndead = 0; + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $tmparray[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 5) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + my $xattr = $values[3]; + my $hostname = $values[1]; + + # count the number of each host by incrementing the hash element corresponding to the hostname: + $hostcount{$hostname} ++; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + if ($xattr > 1) { $Ndead ++; } + } + + # SKIP : already fixed + if (($Nmain == 1) && ($Nback == 1)) { + print STDOUT "GOOD: $summary\n"; + next; + } + + # single copy @ ITC, no backup + # make a backup + if (($Nmain == 1) && ($Nback == 0)) { + print STDOUT "SING: $summary\n"; + my $status = vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } + next; + } + + # ITC files with a single backup copies need to be culled + # cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 1)) { + my $Ncull = $Nmain - 1; + print STDOUT "CULL: $summary\n"; + for (my $j = 0; $j < $Ncull; $j ++) { + my $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } + } + next; + } + + # multiple copies @ ITC + # make a backup and cull Nmain - 1 + if (($Nmain > 1) && ($Nback == 0)) { + my $Ncull = $Nmain - 1; + print STDOUT "DUPS: $summary\n"; + my $status = vsystem("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL REP: neb-replicate : $summary\n"; next; } + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL CUL: neb-cull $j : $summary\n"; next; } + } + next; + } + + if (($Nmain == 0) && ($Nback == 0) && $Ndead) { + print STDOUT "DEAD: $summary\n"; + next; + } + if (($Nmain == 0) && ($Nback == 0) && ($Ndead == 0)) { + print STDOUT "GONE: $summary\n"; + next; + } + + # single copy @ ATRC + # make a backup @ ITC + if (($Nmain == 0) && ($Nback == 1)) { + print STDOUT "SBCK: $summary\n"; + next; + } + + # multiple copies @ ATRC, but none at ITC + # make a backup at ITC, cull ATRC extras + if (($Nmain == 0) && ($Nback > 1)) { + print STDOUT "XBCK: $summary\n"; + next; + } + + # unexpected state: + print STDOUT "OOPS: $summary\n"; + next; +} + +my @hostnames = keys %hostcount; +foreach my $myname (@hostnames) { + print "host $myname = $hostcount{$myname}\n"; +} + +die "done"; + +#my $query = $dbh->prepare( $sql_get_volume_id ); + +#$query->execute( $tgt_volume ); +#my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +#$query->finish; + + +### XX +### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; + +# load the list of files from movelist + +##XX open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +##XX my @movefiles = ; +##XX close (MOVELIST); +##XX +##XX my $otime = $time; +##XX $time = time(); +##XX my $dtime = $time - $otime; +##XX +##XX for (my $i = 0; $i < @movefiles; $i++) { +##XX +##XX my $movefile = $movefiles[$i]; +##XX chomp $movefile; +##XX +##XX # the filename should have the following form: +##XX # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: +##XX # NN : 2-hexdigit name of the subsubdir +##XX # DDDDD : instance ID (long) +##XX # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name +##XX +##XX # I want to check two things: +##XX # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile +##XX # 2) does the vol_id of the instance_id match our vol_id ? +##XX +##XX unless (-e "$tgt_subdir/$movefile") { +##XX print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; +##XX next; +##XX } +##XX +##XX # print "movefile: $movefile\n"; +##XX +##XX my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; +##XX my $summary = "$tgt_subdir/$movefile"; +##XX +##XX if ( ! defined ($ins_id)) { +##XX print "ERROR : INVALID ID: $summary\n"; +##XX next; +##XX } +##XX +##XX if ( $ins_id =~ m/\D/ ) { +##XX print "ERROR : INVALID ID: $summary\n"; +##XX next; +##XX } +##XX $summary = "$summary : $ins_id"; +##XX +##XX # get so_id for this ins_id: +##XX my $so_id = 0; +##XX if (1) { +##XX my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $so_id_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # We should find this instance +##XX print "ERROR : NO DB ENTRY: $summary\n"; +##XX next; +##XX } +##XX +##XX if ($Nrows > 1) { # We should only find one instance +##XX print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; +##XX next; +##XX } +##XX +##XX my $valref = $array[0]; +##XX my @values = @$valref; +##XX +##XX my $Nvalue = @values; +##XX +##XX if ($Nvalue != 1) { +##XX print "ERROR : INVALID DB RESPONSE: $summary\n"; +##XX next; +##XX } +##XX $so_id = $values[0]; +##XX } +##XX +##XX $summary = "$summary : $so_id"; +##XX +##XX # get user_copies for this storage_object +##XX my $Ncopies = 0; +##XX my $ext_id = ""; +##XX if (1) { +##XX my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # user.copies is not set, skip +##XX print "SKIP: NO USER.COPIES ENTRY: $summary\n"; +##XX next; +##XX } +##XX +##XX if ($Nrows > 1) { # We should only find one instance +##XX print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; +##XX next; +##XX } +##XX +##XX my $valref = $array[0]; +##XX my @values = @$valref; +##XX +##XX my $Nvalue = @values; +##XX +##XX if ($Nvalue != 2) { +##XX print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; +##XX next; +##XX } +##XX $ext_id = $values[0]; +##XX $Ncopies = $values[1]; +##XX } +##XX +##XX $summary = "$summary : $ext_id"; +##XX +##XX if ($Ncopies < 2) { +##XX print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; +##XX next; +##XX } +##XX +##XX # check for backup instances +##XX if (1) { +##XX my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; +##XX +##XX my $rowref = $dbh->selectall_arrayref( $ncopies_query ); +##XX my @array = @$rowref; +##XX my $Nrows = @array; +##XX +##XX if ($Nrows == 0) { # We should find this instance +##XX print "REPL: NO BACKUP COPY: $summary\n"; +##XX next; +##XX } +##XX +##XX ## my $valref = $array[0]; +##XX ## my @values = @$valref; +##XX ## +##XX ## my $Nvalue = @values; +##XX ## +##XX ## if ($Nvalue != 1) { +##XX ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; +##XX ## next; +##XX ## } +##XX } +##XX +##XX } # END loop over movefiles + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + +sub vsystem { + # print STDERR "@_\n"; + return 0; + #my $status = system ("@_"); + #return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/create.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/create.rawfix.sh (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/create.rawfix.sh (revision 42399) @@ -0,0 +1,14 @@ +#!/bin/csh + +# make the rawfix database to track rawfix operations +set DBNAME = "rawfix" +set DBUSER = "dvo" + +mysql -h localhost -u root < qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: fix_chip_locations.pl --dateobs (night) + +# we need to select targets for ITC copies of files currently only at ATRC: +# choose a list of the neb hosts with lots of space (by percent or bytes?) +# for each chip, target a single host? + +my ($fulldate, $dateword, $chiplist) = &parse_cmdopts; +my ($neb_dbh) = &parse_db_config ('nebulous.config'); + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# choose ITC machines on which to place backup copies: +my @volarray = &get_neb_volumes; +my $volnum = 0; + +my $Ngood = 0; +my $Ngone = 0; +my $Ndead = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +foreach my $chip_id (@chip_ids) { + + my $urifile = "$dateword/$chip_id.uris.txt"; + open (FILE, "$urifile"); + my @lines = ; + close (FILE); + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + # we expect lines of the form: STATE PATH + # e.g., GOOD: gpc1/20100502/o5318g0002d/o5318g0002d.ota76.fits + if (@words != 2) { die "error in URI list $urifile, line: $line\n"; } + + my $state = $words[0]; + my $ext_id = $words[1]; + + if ($state eq "GOOD:") { $Ngood++; next; } + if ($state eq "DEAD:") { $Ndead++; next; } + if ($state eq "GONE:") { $Ngone++; next; } + if ($state eq "DUPS:") { $Ndups++; next; } + if ($state eq "XBCK:") { $Nxbck++; next; } + + # one copy at ITC: neb-replicate will create a copy at ATRC + if ($state eq "SING:") { + my $status = &vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL SING: neb-replicate : $ext_id\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 1) { die "new instance is not at ATRC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$dateword/$newhost.inst.txt"); + print OUTFILE "$newfile\n"; + close (OUTFILE); + $Nsing ++; + next; + } + + # one copy at ATRC: neb-replicate will create a copy at ITC iff we supply the target volume + if ($state eq "SBCK:") { + my $tgtvol = $volarray[$volnum]; + my $status = &vsystem ("neb-replicate --volume $tgtvol $ext_id"); + if ($status) { print STDERR "FAIL SBCK: neb-replicate : $ext_id on $tgtvol\n"; next; } + my ($newfile, $newhost, $newxattr) = &get_new_instance ($ext_id); + unless ($newxattr == 0) { die "new instance is not at ITC? $ext_id : $newfile : $newhost : $newxattr\n"; } + open (OUTFILE, ">>$dateword/$newhost.inst.txt"); + print OUTFILE "$newfile\n"; + close (OUTFILE); + $Nsbck ++; + next; + } + } + $volnum ++; +} + +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; + +die "all done"; + +sub get_new_instance { + + my $ext_id = $_[0]; + + ## the first instance should be the last one created: + my $sql_chip_instances = "SELECT uri, host, xattr from instance join storage_object using (so_id) join volume using (vol_id) where ext_id = '$ext_id' order by ins_id desc"; + my $rowref = $neb_dbh->selectall_arrayref( $sql_chip_instances ); + my @tmparray = @$rowref; + + my $valref = $tmparray[0]; + my @values = @$valref; + + my $uri = $values[0]; + my $host = $values[1]; + my $xattr = $values[2]; + + return ($uri, $host, $xattr); +} + +# get the list of chip neb paths for one chip +sub get_neb_volumes { + + my @volarray; + + my $sql_neb_volumes = "select name from (select name, host, allocate, available, xattr, total, used, total - used as unused from mountedvol where (xattr = 0) and (allocate = 1) and (available = 1)) as tmp order by unused desc"; + if ($DEBUG > 4) { print "sql: $sql_neb_volumes\n"; } + + my $rowref = $neb_dbh->selectall_arrayref( $sql_neb_volumes ); + my @tmparray = @$rowref; + my $Nrows = @tmparray; + + if ($DEBUG > 4) { print "found $Nrows possible volumes\n"; } + + foreach my $valref (@tmparray) { + my @values = @$valref; + # there should only be a single value (only name selected above) + my $Nvalues = @values; + if ($Nvalues == 0) { print "ERROR: missing uri\n"; next; } + if ($Nvalues > 1) { print "ERROR: too many uri entries?\n"; next; } + + push (@volarray, $values[0]); + if ($DEBUG > 4) { print "$values[0]\n"; } + } + + return @volarray; +} + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,$chiplist) or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $nebdb_name, $nebdb_host, $nebdb_port, $nebdb_pass, $nebdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /NEB_NAME /) { $nebdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $nebdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $nebdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $nebdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $nebdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "nebulous config: $nebdb_name $nebdb_host $nebdb_port $nebdb_user $nebdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure nebulous database (dbname)", -exitval => 2) unless defined $nebdb_name; + pod2usage( -msg => "Cannot configure nebulous database (dbhost)", -exitval => 2) unless defined $nebdb_host; + pod2usage( -msg => "Cannot configure nebulous database (dbuser)", -exitval => 2) unless defined $nebdb_user; + pod2usage( -msg => "Cannot configure nebulous database (dbport)", -exitval => 2) unless defined $nebdb_port; + pod2usage( -msg => "Cannot configure nebulous database (dbpass)", -exitval => 2) unless defined $nebdb_pass; + + my $nebdb_dsn = "DBI:mysql:database=$nebdb_name:host=$nebdb_host"; + if ($nebdb_port ne "NONE") { $nebdb_dsn .= ":port=$nebdb_port"; } + my $neb_dbh = DBI->connect($nebdb_dsn, $nebdb_user, $nebdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($neb_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/get_md5s_dateobs.pl (revision 42399) @@ -0,0 +1,50 @@ +#! /usr/bin/env perl + +die "XXX not done yet"; + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_dateobs.pl --dateobs (night) + +my ($fulldate, $dateword) = &parse_cmdopts; + +# first, ensure the dateobs directory is + + + +sub parse_cmdopts { + + my ($dateobs); + GetOptions( 'dateobs=s' => \$dateobs) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/get_md5s_instances.pl (revision 42399) @@ -0,0 +1,87 @@ +#!/usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: get_md5s_instances.pl --hostname (hostname) --input (file) + +my ($myhost, $input) = &parse_cmdopts; + +# check that we are running this script on 'myhost' +my $truhost = `hostname -s`; chomp $truhost; +if ($truhost ne $myhost) { die "must run on $myhost (not $truhost)\n"; } + +# read the list of instances: +my @ins_list = &load_instance_list ($input); + +my $output = $input; +$output =~ s/inst/md5s/; + +if ($output eq $input) { die "output file ($output) matches input file ($input)\n"; } +open (OUTFILE, ">$output"); + +foreach my $instance (@ins_list) { + + my $filename = $instance; + $filename =~ s|file://||; + # print "filename: $filename\n"; + + my $line = `md5sum $filename`; + + my ($md5sum, $rawfile) = split (" ", $line); + # print "md5sum: $md5sum\n"; + + print OUTFILE "$instance $md5sum\n"; +} +close OUTFILE; + +die "all done"; + +# read the list of chip_ids: +sub load_instance_list { + my $infile = $_[0]; + + my @ins_list; + + open(FILE,$infile) or die "cannot open file $infile\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in input $infile : $line\n"; } + push (@ins_list, $words[0]); + } + close (FILE); + + return @ins_list; +} + +sub parse_cmdopts { + + my ($myhost, $input); + GetOptions( 'hostname=s' => \$myhost, 'input=s' => \$input) || pod2usage(2); + + pod2usage( -msg => "Provide the input list with --input", -exitval => 2) unless defined $input; + pod2usage( -msg => "Define the host machine with --hostname", -exitval => 2) unless defined $myhost; + + return ($myhost, $input); +} + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/neb_check_location.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/neb_check_location.pl (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/neb_check_location.pl (revision 42399) @@ -0,0 +1,244 @@ +#! /usr/bin/env perl + +# TESTRUN can be 1, 2, 3, 4 to get further in the tests +my $TESTRUN = 2; + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($tgthost,$tgtvol,$subdir,$movelist); + +GetOptions( + 'tgthost=s' => \$tgthost, + 'tgtvol=s' => \$tgtvol, + 'subdir=s' => \$subdir, + 'list=s' => \$movelist) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = 'nebulous.config'; + +if (-e $conffile) { +# printf STDERR "found config\n"; + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine target machine and volume", -exitval => 2) unless + defined $tgthost && defined $tgtvol; + +pod2usage( -msg => "Cannot determine subdir or list of files which have been moved", -exitval => 2) unless + defined $movelist && defined $subdir; + +my $host_check = `hostname`; chomp($host_check); + +if ($host_check ne $tgthost) { die "Required to run on $tgthost, not on $host_check\n"; } + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# example: +# tgthost : ipp130 +# tgtvol : 0 + +# Get the vol_id of the (local) destination. + +my $tgt_volume = "${tgthost}.${tgtvol}"; +my $tgt_subdir = "/export/${tgthost}.${tgtvol}/nebulous/${subdir}"; + +### XX my $sql_get_volume_id = "SELECT vol_id, name, host, path FROM volume WHERE name = ?"; +### XX my $sql_update_instance = "UPDATE instance SET vol_id = ?, uri = ? WHERE ins_id = ?"; +### XX +### XX my $query = $dbh->prepare( $sql_get_volume_id ); +### XX +### XX $query->execute( $tgt_volume ); +### XX my ($tgt_vol_id, $tgt_vol_name, $tgt_vol_host, $tgt_vol_path) = $query->fetchrow_array; +### XX $query->finish; +### XX +### XX print "LOG INIT $tgthost $tgtvol $tgt_vol_id\n"; + +# load the list of files from movelist + +open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +my @movefiles = ; +close (MOVELIST); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; + +for (my $i = 0; $i < @movefiles; $i++) { + + my $movefile = $movefiles[$i]; + chomp $movefile; + + # the filename should have the following form: + # NN/DDDDDDDDDD.WWW:WWW:WWWW:WWW: + # NN : 2-hexdigit name of the subsubdir + # DDDDD : instance ID (long) + # WWW:WWW:WWW : elements of the filename, with : replacing / in the neb storage_object name + + # I want to check two things: + # 1) does the file exist at /data/$tgthost.$tgtvol/nebulous/$subdir/$movefile + # 2) does the vol_id of the instance_id match our vol_id ? + + unless (-e "$tgt_subdir/$movefile") { + print "ERROR : MISSING FILE: $tgt_subdir/$movefile\n"; + next; + } + + # print "movefile: $movefile\n"; + + my ($ins_id) = $movefile =~ m|^[0-9a-fA-F][0-9a-fA-F]/(\d+)\..*|; + my $summary = "$tgt_subdir/$movefile"; + + if ( ! defined ($ins_id)) { + print "ERROR : INVALID ID: $summary\n"; + next; + } + + if ( $ins_id =~ m/\D/ ) { + print "ERROR : INVALID ID: $summary\n"; + next; + } + $summary = "$summary : $ins_id"; + + # get so_id for this ins_id: + my $so_id = 0; + if (1) { + my $so_id_query = "SELECT so_id FROM instance WHERE ins_id = ${ins_id}"; + + my $rowref = $dbh->selectall_arrayref( $so_id_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "ERROR : NO DB ENTRY: $summary\n"; + next; + } + + if ($Nrows > 1) { # We should only find one instance + print "ERROR : TOO MANY SO_ID ENTRIES: $summary\n"; + next; + } + + my $valref = $array[0]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 1) { + print "ERROR : INVALID DB RESPONSE: $summary\n"; + next; + } + $so_id = $values[0]; + } + + $summary = "$summary : $so_id"; + + # get user_copies for this storage_object + my $Ncopies = 0; + my $ext_id = ""; + if (1) { + my $ncopies_query = "SELECT ext_id, value FROM storage_object JOIN storage_object_xattr USING (so_id) WHERE (so_id = ${so_id}) and (name = 'user.copies')"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # user.copies is not set, skip + print "SKIP: NO USER.COPIES ENTRY: $summary\n"; + next; + } + + if ($Nrows > 1) { # We should only find one instance + print "ERROR : TOO MANY XATTR ENTRIES: $summary : $Nrows\n"; + next; + } + + my $valref = $array[0]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 2) { + print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + $ext_id = $values[0]; + $Ncopies = $values[1]; + } + + $summary = "$summary : $ext_id"; + + if ($Ncopies < 2) { + print "SKIP: FEWER THAN 2 USER.COPIES REQUESTED: $summary : $Ncopies\n"; + next; + } + + # check for backup instances + if (1) { + my $ncopies_query = "SELECT ins_id, vol_id FROM instance JOIN volume using (vol_id) WHERE (so_id = ${so_id}) and (xattr = 1)"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find this instance + print "REPL: NO BACKUP COPY: $summary\n"; + next; + } + + ## my $valref = $array[0]; + ## my @values = @$valref; + ## + ## my $Nvalue = @values; + ## + ## if ($Nvalue != 1) { + ## print "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + ## next; + ## } + } + +} # END loop over movefiles + +# example of using the return value of arrayref: + +# my $rowref = $dbh->selectall_arrayref( $validate_ins_id_query ); +# my @array = @$rowref; +# my $Nrows = @array; + +# print "NROWS: $Nrows\n"; + +# foreach my $valref (@array) { +# my @values = @$valref; +# foreach my $value (@values) { +# print "VAL: $value, "; +# } +# print "\n"; +# } + Index: /trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh (revision 42399) @@ -0,0 +1,17 @@ +#!/bin/csh + +if ($#argv != 1) then + echo "USAGE: queue.rawfix.sh (mjddate)" + exit 2 +endif + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +## XXX validate the format (YYYYMMDD)? +set mynight = $1 + +mysql -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME -e "insert into nights (dateobs, state) values ('$mynight', 'new')" Index: /trunk/tools/eam/rawfix.20230221/src/triple.fix.pl =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/triple.fix.pl (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/triple.fix.pl (revision 42399) @@ -0,0 +1,196 @@ +#! /usr/bin/env perl + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +my ($movelist); + +GetOptions( 'list=s' => \$movelist) || pod2usage(2); + +my ( $dbname, $dbhost, $dbport, $dbpass, $dbuser ); +my $conffile = '/home/panstarrs/eugene/triples.20201211/nebulous.config'; + +if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /NEB_NAME /) { $dbname = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_HOST /) { $dbhost = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_USER /) { $dbuser = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PASS /) { $dbpass = (split /\s+/,$_)[2]; } + if ($_ =~ /NEB_PORT /) { $dbport = (split /\s+/,$_)[2]; } + } + close(IN); + # print STDERR "$dbname $dbhost $dbport $dbuser $dbpass\n"; +} + +pod2usage( -msg => "Cannot configure nebulous database", -exitval => 2) unless + defined $dbname && defined $dbhost && defined $dbport && defined $dbuser && defined $dbpass; + +pod2usage( -msg => "Cannot determine list of files which have been moved", -exitval => 2) unless + defined $movelist; + +my $time = time(); + +my $dsn; +if ($dbport eq "NONE") { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost"; +} else { + $dsn = "DBI:mysql:database=$dbname:host=$dbhost:port=$dbport"; +} + +my $dbh = DBI->connect($dsn, $dbuser, $dbpass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + +# load the list of files from movelist + +# the movelist files generated by 'find_triples.pl' has three header lines +# followed by a list of ext_id only (no so_id) + +open (MOVELIST, $movelist) or die "cannot open list of files $movelist\n"; +my @movefiles = ; +close (MOVELIST); + +my $otime = $time; +$time = time(); +my $dtime = $time - $otime; + +# skip the first 3 header lines +for (my $i = 3; $i < @movefiles; $i++) { + + my $movefile = $movefiles[$i]; + chomp $movefile; + + my @word = split (" ", $movefile); + + my $so_id; + my $ext_id; + if (0) { + # output from rebalance code has so_id ext_id, but not find_triples.pl + # so_id ext_id + $so_id = $word[0]; + $ext_id = $word[1]; + } else { + $so_id = 0; + $ext_id = $word[0]; + } + + # if ( ! defined ($so_id)) { + # print STDERR "ERROR : INVALID SO_ID: $movefile\n"; + # next; + # } + + if ( ! defined ($ext_id)) { + print STDERR "ERROR : MISSING EXT_ID: $movefile\n"; + next; + } + + #### LIMIT to FITS, SMF, CMF #### + # for now, only do *.fits files: + my $is_valid = 0; + if ($ext_id =~ m|fits$|) { $is_valid = 1; } + if ($ext_id =~ m|cmf$|) { $is_valid = 1; } + if ($ext_id =~ m|smf$|) { $is_valid = 1; } + + # skip files which do not match desired rules: + unless ($is_valid) { next; } + + # my $summary = "$ext_id $so_id"; + my $summary = "$ext_id"; + + # get user_copies for this storage_object + my $Nmain = 0; + my $Nback = 0; + if (1) { + my $ncopies_query = "SELECT ins_id, vol_id, xattr FROM instance JOIN volume USING (vol_id) JOIN storage_object using (so_id) WHERE (ext_id = '${ext_id}')"; + + my $rowref = $dbh->selectall_arrayref( $ncopies_query ); + my @array = @$rowref; + my $Nrows = @array; + + if ($Nrows == 0) { # We should find an instance + print STDERR "ERROR: NO INSTANCES: $summary\n"; + next; + } + + # xattr = 0 means instance @ ITC (main) + # xattr = 1 means instance @ ATRC (back) + ## how many xattr = 0 and how many xattr = 1 + for (my $j = 0; $j < $Nrows; $j ++) { + my $valref = $array[$j]; + my @values = @$valref; + + my $Nvalue = @values; + + if ($Nvalue != 3) { + print STDERR "ERROR : INVALID DB XATTR RESPONSE: $summary\n"; + next; + } + my $xattr = $values[2]; + + if ($xattr == 0) { $Nmain ++; } + if ($xattr == 1) { $Nback ++; } + } + } + $summary = $summary . " $Nmain $Nback"; + + # SKIP : already fixed + if (($Nback == 1) && ($Nmain == 1)) { + print STDERR "SKIP 1: BACKUP AND MAIN OK: $summary\n"; + next; + } + + # SKIP : too many backups, how to cull them? + if ($Nback > 1) { + print STDERR "SKIP 2: TOO MANY BACKUPS : $summary\n"; + next; + } + + # ITC files with a single backup copies need to be culled + # this case was SKIP 3: + if (($Nback == 1) && ($Nmain > 1)) { + my $Ncull = $Nmain - 1; + print STDERR "CULL: TOO MANY MAIN BUT BACKUP OK, CULL $Ncull X: $summary\n"; + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; } + } + next; + } + + # single copy @ ITC + if (($Nmain == 1) && ($Nback == 0)) { + print STDERR "SING: MAKE BACKUP : $summary\n"; + my $status = 0; + $status = vsystem ("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL SN1: neb-replicate : $summary\n"; next; } + next; + } + + # multiple copies @ ITC + if (($Nmain > 1) && ($Nback == 0)) { + my $Ncull = $Nmain - 1; + print STDERR "DUPS: MAKE BACKUP AND CULL $Ncull X : $summary\n"; + my $status = 0; + $status = vsystem("neb-replicate $ext_id"); + if ($status) { print STDERR "FAIL TR1: neb-replicate : $summary\n"; next; } + for (my $j = 0; $j < $Ncull; $j ++) { + $status = vsystem("neb-cull $ext_id"); + if ($status) { print STDERR "FAIL TRN: neb-cull $j : $summary\n"; next; } + } + next; + } + + print STDERR "ERROR: NO BACKUP AND NO MAIN: $summary\n"; +} # END loop over movefiles + +sub vsystem { + print STDERR "@_\n"; + #return 0; + my $status = system ("@_"); + return $status; +} Index: /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh =================================================================== --- /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh (revision 42399) +++ /trunk/tools/eam/rawfix.20230221/src/update.rawfix.sh (revision 42399) @@ -0,0 +1,21 @@ +#!/bin/csh + +if ($#argv < 1) then + echo "USAGE: update.rawfix.sh (night) (state)" + echo " EG: update.rawfix.sh 20100501 new" + exit 2 +endif + +set mynight = $1 +set state = $2 + +# make the rawfix database to track rawfix operations +set DBHOST = "ipp060" +set DBNAME = "rawfix" +set DBUSER = "dvo" +set DBPASS = "dvo" + +set DBOPTS = "-B --skip-column-names -h $DBHOST -u $DBUSER -p$DBPASS $DBNAME" + +mysql $DBOPTS -e "update nights set state = '$state' where (dateobs = '$mynight')" +exit 0