Index: trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl (revision 42447) +++ trunk/tools/eam/rawfix.20230221/src/advance.rawfix.pl (revision 42449) @@ -144,4 +144,8 @@ my $Nfail = $values[14]; + # in state of 'new', we do not care what the values of the stat fields are. + # they may be NULL, so do not yet try to do math on them here + if ($state eq "new") { &update_night_state ($dateobs, "ckchip.new", 0); return "NEW"; } + my $Ndone = $Ngood + $Ndead + $Ngone + $Nfail; my $Nwait = $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3; @@ -151,5 +155,4 @@ # check for completion here first (otherwise, advance fixchip.done below) if (($state eq "fixchip.done") and ($iter > 0) and ($Nwait == 0)) { &update_night_state ($dateobs, "DONE"); return "DONE"; } - if ($state eq "new") { &update_night_state ($dateobs, "ckchip.new", 0); return "NEW"; } if ($state eq "ckchip.done") { &update_night_state ($dateobs, "fixchip.new"); return "NEW"; } if ($state eq "fixchip.done") { &update_night_state ($dateobs, "md5sum.new"); return "NEW"; } Index: trunk/tools/eam/rawfix.20230221/src/finish.fixchip.sh =================================================================== --- trunk/tools/eam/rawfix.20230221/src/finish.fixchip.sh (revision 42449) +++ trunk/tools/eam/rawfix.20230221/src/finish.fixchip.sh (revision 42449) @@ -0,0 +1,35 @@ +#!/bin/csh + +if ($#argv != 4) then + echo "USAGE: finish.fixchip.sh (topdir) (dateobs) (iter) (QUALITY)" + exit 2 +endif + +set topdir = $1 +set dateobs = $2 +set iter = $3 +set QUALITY = $4 + +if (! -d $topdir) then + echo "ERROR: missing top-level directory $topdir" + exit 3 +endif + +if (! -d $topdir/$dateobs) then + echo "ERROR: missing dateobs directory $topdir/$dateobs" + exit 3 +endif + +set fulldate = `echo $dateobs | awk '{printf "%s/%s/%s\n", substr($1,1,4), substr($1,5,2), substr($1,7,2)}'` +echo $fulldate + +$topdir/update_chip_stats.pl --topdir $topdir --dateobs $fulldate --stage $iter --chiplist gpc1.chips.txt + +if ($QUALITY == "OK") then + $topdir/update.rawfix.sh $dateobs fixchip.done +else + $topdir/update.rawfix.sh $dateobs fixchip.fail +endif + +exit 0 + Index: trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42447) +++ trunk/tools/eam/rawfix.20230221/src/fix_chip_locations.pl (revision 42449) @@ -12,5 +12,5 @@ use Getopt::Long qw( GetOptions :config auto_help auto_version ); -# USAGE: fix_chip_locations.pl --dateobs (night) --chiplist (file) --stage (0,1,etc) +# USAGE: fix_chip_locations.pl --dateobs (night) --stage (0,1,etc) [--chiplist (file) | --chipID (ID)] # we need to select targets for ITC copies of files currently only at ATRC: @@ -18,14 +18,10 @@ # for each chip, target a single host? -my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir) = &parse_cmdopts; my ($neb_dbh, $raw_dbh) = &parse_db_config ("$topdir/nebulous.config"); - -# read the list of chip_ids: -my @chip_ids = &load_chip_list ($chiplist); # choose ITC machines on which to place backup copies: my @vol_copy = &get_neb_volumes_copy; # array of neb hosts to copy data to at ITC my %vol_good = &get_neb_volumes_good; # hash of valid neb hosts -my $volnum = 0; my $Ngood = 0; @@ -41,4 +37,6 @@ my $Ncul3 = 0; my $Nfail = 0; + +my $Nchip = 0; # counts the number of instances # load the lists of good and fail chips @@ -69,7 +67,66 @@ } -my $Nchip = 0; - +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +my $volnum = 0; # select the volume to be used from the list of @vol_copy + +# loop over all chip_ids so we can distribute the target volumes foreach my $chip_id (@chip_ids) { + if (not defined $chipID or ($chipID eq $chip_id)) { + &fix_chip_locations_chipID ($chip_id, $volnum); + } + $volnum ++; + if ($volnum > @vol_copy) { $volnum = 0; } +} + +my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail; +if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } + +if (not defined $chipID) { + # update the stats in the database + my $sql_update_stats = "UPDATE nights set"; + $sql_update_stats .= " Nchip = $Nchip"; + $sql_update_stats .= ", Ngood = $Ngood"; + $sql_update_stats .= ", Ndead = $Ndead"; + $sql_update_stats .= ", Ngone = $Ngone"; + $sql_update_stats .= ", Nsing = $Nsing"; + $sql_update_stats .= ", Ndups = $Ndups"; + $sql_update_stats .= ", Nsbck = $Nsbck"; + $sql_update_stats .= ", Nxbck = $Nxbck"; + $sql_update_stats .= ", Ncul1 = $Ncul1"; + $sql_update_stats .= ", Ncul2 = $Ncul2"; + $sql_update_stats .= ", Ncul3 = $Ncul3"; + $sql_update_stats .= ", Nfail = $Nfail"; + $sql_update_stats .= " WHERE dateobs = $dateword"; + + my $query = $raw_dbh->prepare($sql_update_stats); + $query->execute () or die "invalid response from RAW database\n"; +} else { + + open (OUTFILE, ">$topdir/$dateword/$chipID.stats.$stage.txt"); + + print OUTFILE "Nchip: $Nchip\n"; + print OUTFILE "Ngood: $Ngood\n"; + print OUTFILE "Ngone: $Ngone\n"; + print OUTFILE "Ndead: $Ndead\n"; + print OUTFILE "Nsing: $Nsing\n"; + print OUTFILE "Ndups: $Ndups\n"; + print OUTFILE "Nsbck: $Nsbck\n"; + print OUTFILE "Nxbck: $Nxbck\n"; + print OUTFILE "Ncul1: $Ncul1\n"; + print OUTFILE "Ncul2: $Ncul2\n"; + print OUTFILE "Ncul3: $Ncul3\n"; + print OUTFILE "Nfail: $Nfail\n"; + + close (OUTFILE); +} + +exit 0; + +sub fix_chip_locations_chipID { + + my $chip_id = $_[0]; + my $volnum = $_[1]; # read from chip_id file, update to next stage number @@ -216,43 +273,5 @@ } } - $volnum ++; -} - -print "Nchip: $Nchip\n"; -print "Ngood: $Ngood\n"; -print "Ngone: $Ngone\n"; -print "Ndead: $Ndead\n"; -print "Nsing: $Nsing\n"; -print "Ndups: $Ndups\n"; -print "Nsbck: $Nsbck\n"; -print "Nxbck: $Nxbck\n"; -print "Ncul1: $Ncul1\n"; -print "Ncul2: $Ncul2\n"; -print "Ncul3: $Ncul3\n"; -print "Nfail: $Nfail\n"; - -my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail; -if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } - -# update the stats in the database -my $sql_update_stats = "UPDATE nights set"; -$sql_update_stats .= " Nchip = $Nchip"; -$sql_update_stats .= ", Ngood = $Ngood"; -$sql_update_stats .= ", Ndead = $Ndead"; -$sql_update_stats .= ", Ngone = $Ngone"; -$sql_update_stats .= ", Nsing = $Nsing"; -$sql_update_stats .= ", Ndups = $Ndups"; -$sql_update_stats .= ", Nsbck = $Nsbck"; -$sql_update_stats .= ", Nxbck = $Nxbck"; -$sql_update_stats .= ", Ncul1 = $Ncul1"; -$sql_update_stats .= ", Ncul2 = $Ncul2"; -$sql_update_stats .= ", Ncul3 = $Ncul3"; -$sql_update_stats .= ", Nfail = $Nfail"; -$sql_update_stats .= " WHERE dateobs = $dateword"; - -my $query = $raw_dbh->prepare($sql_update_stats); -$query->execute () or die "invalid response from RAW database\n"; - -exit 0; +} sub get_new_instance { @@ -412,12 +431,12 @@ sub parse_cmdopts { - my ($dateobs, $chiplist, $stage, $topdir); - GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + my ($dateobs, $chiplist, $chipID, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'chipID=s' => \$chipID, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; - pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; - + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + # check that stage is an integer unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } @@ -435,5 +454,5 @@ my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); - return ($fulldate, $dateword, $chiplist, $stage, $topdir); + return ($fulldate, $dateword, $chiplist, $chipID, $stage, $topdir); } Index: trunk/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh =================================================================== --- trunk/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh (revision 42449) +++ trunk/tools/eam/rawfix.20230221/src/get_hosts_fixchip.sh (revision 42449) @@ -0,0 +1,34 @@ +#!/bin/csh + +if ($#argv != 3) then + echo "USAGE: get_hosts_fixchip.sh (topdir) (dateobs) (iter)" + exit 2 +endif + +set topdir = $1 +set dateobs = $2 +set iter = $3 + +if (! -d $topdir) then + echo "ERROR: missing top-level directory $topdir" + exit 3 +endif + +if (! -d $topdir/$dateobs) then + echo "ERROR: missing dateobs directory $topdir/$dateobs" + exit 3 +endif + +# report both the dateobs and the hostnames +echo "dateobs chipID iter fixhost" + +foreach f ($topdir/$dateobs/XY??.hosts.txt) + set myhost = `cat $f | grep -v ippb | sort -k 2 | tail -n 1 | prcol 1` + set chipID = `echo $f | awk -F/ '{print $NF}' | awk -F. '{print $1}'` + if ("$myhost" == "") then + set myhost = ipp060 + endif + echo $dateobs $chipID $iter $myhost +end + +## XXX ipp060 is hardwired for no good reason Index: trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh =================================================================== --- trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh (revision 42447) +++ trunk/tools/eam/rawfix.20230221/src/queue.rawfix.sh (revision 42449) @@ -29,7 +29,4 @@ if ($mode == "night") then ## XXX validate the format (YYYYMMDD)? - - XXXXX: - set mynight = $2 Index: trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt (revision 42447) +++ trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt (revision 42449) @@ -9,4 +9,5 @@ if (not($?RAWFIX_FIXCHIP_INIT)) book init fixchip.book + book init fixhost.book $RAWFIX_FIXCHIP_INIT = 1 end @@ -27,5 +28,4 @@ task.exec - # command check.rawfix.sh state ckchip.done command check.rawfix.sh state fixchip.new end @@ -53,10 +53,16 @@ end -task nights.fixchip.run +# output: fixhost.book +task nights.fixchip.loadhosts periods -poll $RPOLL periods -exec $REXEC - periods -timeout 1800 - host -required $HOST_FIXCHIP + periods -timeout 60 + host local npending 1 + # note: the tasks which return their results via stdout + # seem to fail if run remotely. + + stdout $LOGDIR/fixchip.loadhosts.stdout.log + stderr $LOGDIR/fixchip.loadhosts.stderr.log task.exec @@ -76,7 +82,73 @@ book getword fixchip.book $pageName iter -var ITER option $pageName - - stdout $pageName/log.fixchip.v$ITER - stderr $pageName/log.fixchip.v$ITER + # save the page so we can use it below + + # write an entry point in the log for this night + sprintf tmpline "%s/%s/log.fixchip.runhosts.v%d" $mypath $DATEOBS $ITER + date -var mydate + exec echo "# start runhosts $mydate" >> $tmpline + + # the command returns these fields: dateobs stage fixhost chipID + # NOTE: stage == ITER + command $mypath/get_hosts_fixchip.sh $mypath $DATEOBS $ITER + end + + task.exit 0 + # convert the stdout 'queue' to entries in a book + # fields in the table will become words in the page + queueprint stdout + queuesize stdout -var Nhosts + queue2book -raw-columns -key dateobs:chipID -uniq -setword pantaskState INIT stdout fixhost.book + book setword fixchip.book $options:0 NHOST {$Nhosts - 1} + book setword fixchip.book $options:0 NDONE 0 + book list + book listbook fixchip.book + # use the queuesize (Nhosts) to track how many jobs are done / to be done + end + + # exit values other than 0: + task.exit default + showcommand failure + book setword fixchip.book $options:0 pantasksQuality FAIL + end + + task.exit crash + showcommand crash + book setword fixchip.book $options:0 pantasksQuality CRASH + end + + # operation times out? + task.exit timeout + showcommand timeout + book setword fixchip.book $options:0 pantasksQuality TIMEOUT + end +end + +task nights.fixchip.run + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 1800 + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixhost.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for new nights in fixhost.book (pantaskState == INIT) + book getpage fixhost.book 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + book setword fixhost.book $pageName pantaskState RUN + book getword fixhost.book $pageName dateobs -var DATEOBS + book getword fixhost.book $pageName iter -var ITER + book getword fixhost.book $pageName fixhost -var FIXHOST + book getword fixhost.book $pageName chipID -var CHIP_ID + option $pageName $DATEOBS + + stdout $pageName/log.fixchip.run.v$ITER + stderr $pageName/log.fixchip.run.v$ITER substr $DATEOBS 0 4 YEAR @@ -85,11 +157,85 @@ sprintf FULLDATE "%s/%s/%s" $YEAR $MONTH $DAY - command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --stage $ITER - end - - ## XX delete the page from the book if done? - task.exit 0 + host -required $FIXHOST + ## check if this host needs to be launched + if (not($?host:$FIXHOST)) + echo "host not defined, creating : $FIXHOST" + $host:$FIXHOST = 0 + end + if (not($host:$FIXHOST)) + control host add $FIXHOST + $host:$FIXHOST = 1 + end + + # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs + periods -exec 0.05 + command $mypath/fix_chip_locations.pl --topdir $mypath --dateobs $FULLDATE --chiplist gpc1.chips.txt --chipID $CHIP_ID --stage $ITER + end + + task.exit 0 + nights.fixchip.exit OK + end + + # exit values other than 0: + task.exit default + nights.fixchip.exit FAIL + showcommand failure + end + + task.exit crash + nights.fixchip.exit CRASH + showcommand crash + end + + # operation times out? + task.exit timeout + nights.fixchip.exit TIMEOUT + showcommand timeout + end +end + +# output: fixhost.book +task nights.fixchip.clean + + periods -poll $RPOLL + periods -exec $REXEC + periods -timeout 60 + npending 1 + + stdout $LOGDIR/fixchip.clean.stdout.log + stderr $LOGDIR/fixchip.clean.stderr.log + + task.exec + # if we are unable to run the 'exec', use a long retry time + periods -exec $REXEC + + book npages fixchip.book -var N + if ($N == 0) break + if ($NETWORK == 0) break + + # look for completed nights in fixchip.book (pantaskState == STOP) + book getpage fixchip.book 0 -var pageName -key pantaskState STOP + if ("$pageName" == "NULL") break + + # book listpage fixchip.book $pageName + + # book setword fixchip.book $pageName pantaskState RUN + book getword fixchip.book $pageName dateobs -var DATEOBS + book getword fixchip.book $pageName pantasksQuality -var QUALITY + + option $pageName + # save the page so we can use it below + + if ("$QUALITY" == "OK") + command update.rawfix.sh $DATEOBS fixchip.done + else + command update.rawfix.sh $DATEOBS fixchip.fail + end + end + + task.exit 0 + # now that the entry is set to done in the db, remove the pantasks book entry + echo "cleaning this page: $options:0" book setword fixchip.book $options:0 pantaskState DONE - exec update.rawfix.sh $options:0 fixchip.done end @@ -98,5 +244,4 @@ showcommand failure book setword fixchip.book $options:0 pantasksQuality FAIL - exec update.rawfix.sh $options:0 fixchip.fail end @@ -104,5 +249,4 @@ showcommand crash book setword fixchip.book $options:0 pantasksQuality CRASH - exec update.rawfix.sh $options:0 fixchip.fail end @@ -111,6 +255,32 @@ showcommand timeout book setword fixchip.book $options:0 pantasksQuality TIMEOUT - exec update.rawfix.sh $options:0 fixchip.fail - end -end - + end +end + +######################### UTILITIES ################################# + +macro nights.fixchip.exit + if ($0 != 2) + echo "USAGE: nights.fixchip.exit (quality)" + break + end + + book setword fixhost.book $options:0 pantaskState DONE + book getword fixchip.book $options:1 NHOST -var Nhosts + book getword fixchip.book $options:1 NDONE -var Ndone + book getword fixchip.book $options:1 stage -var ITER + $Ndone ++ + book setword fixchip.book $options:1 NDONE $Ndone + if ("$1" != "OK") + book setword fixchip.book $options:1 pantasksQuality $1 + sprintf tmpline "%s/%s/log.fixchip.runhosts.v%d" $mypath $options:1 $ITER + exec echo "failure $1 $options:0" >> $tmpline + else + exec echo "success $1 $options:0" >> $tmpline + end + if ($Ndone == $Nhosts) + book setword fixchip.book $options:1 pantaskState STOP + end + book delpage fixhost.book $options:0 +end + Index: trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt =================================================================== --- trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt (revision 42447) +++ trunk/tools/eam/rawfix.20230221/src/rawfix.md5sum.pt (revision 42449) @@ -127,5 +127,5 @@ # runs on the hosts identified by get_hosts_md5s.sh # output: md5host.book -task nights.md5sum.runhosts +task nights.md5sum.run periods -poll $RPOLL Index: trunk/tools/eam/rawfix.20230221/src/update_chip_stats.pl =================================================================== --- trunk/tools/eam/rawfix.20230221/src/update_chip_stats.pl (revision 42449) +++ trunk/tools/eam/rawfix.20230221/src/update_chip_stats.pl (revision 42449) @@ -0,0 +1,204 @@ +#! /usr/bin/env perl + +my $DEBUG = 4; +my $time = time(); + +use DBI; +use Carp; +use Pod::Usage qw( pod2usage ); + +use strict; +use warnings FATAL => qw( all ); +use Getopt::Long qw( GetOptions :config auto_help auto_version ); + +# USAGE: update_chip_locations.pl --dateobs (night) --stage (0,1,etc) --chiplist (file) + +my ($fulldate, $dateword, $chiplist, $stage, $topdir) = &parse_cmdopts; +my ($raw_dbh) = &parse_db_config ("$topdir/nebulous.config"); + +my $Ngood = 0; +my $Ndead = 0; +my $Ngone = 0; +my $Nsing = 0; +my $Ndups = 0; +my $Nsbck = 0; +my $Nxbck = 0; + +my $Ncul1 = 0; +my $Ncul2 = 0; +my $Ncul3 = 0; +my $Nfail = 0; + +my $Nchip = 0; # counts the number of instances + +# read the list of chip_ids: +my @chip_ids = &load_chip_list ($chiplist); + +# loop over all chip_ids & read in stats +foreach my $chip_id (@chip_ids) { + + open (STATFILE, "$topdir/$dateword/$chip_id.stats.$stage.txt"); + my @lines = ; + close (STATFILE); + + my $N = 0; + $Nchip += &parse_line ($lines[$N], "Nchip:"); $N ++; + $Ngood += &parse_line ($lines[$N], "Ngood:"); $N ++; + $Ngone += &parse_line ($lines[$N], "Ngone:"); $N ++; + $Ndead += &parse_line ($lines[$N], "Ndead:"); $N ++; + $Nsing += &parse_line ($lines[$N], "Nsing:"); $N ++; + $Ndups += &parse_line ($lines[$N], "Ndups:"); $N ++; + $Nsbck += &parse_line ($lines[$N], "Nsbck:"); $N ++; + $Nxbck += &parse_line ($lines[$N], "Nxbck:"); $N ++; + $Ncul1 += &parse_line ($lines[$N], "Ncul1:"); $N ++; + $Ncul2 += &parse_line ($lines[$N], "Ncul2:"); $N ++; + $Ncul3 += &parse_line ($lines[$N], "Ncul3:"); $N ++; + $Nfail += &parse_line ($lines[$N], "Nfail:"); $N ++; +} + +print "Nchip: $Nchip\n"; +print "Ngood: $Ngood\n"; +print "Ngone: $Ngone\n"; +print "Ndead: $Ndead\n"; +print "Nsing: $Nsing\n"; +print "Ndups: $Ndups\n"; +print "Nsbck: $Nsbck\n"; +print "Nxbck: $Nxbck\n"; +print "Ncul1: $Ncul1\n"; +print "Ncul2: $Ncul2\n"; +print "Ncul3: $Ncul3\n"; +print "Nfail: $Nfail\n"; + +my $Ntotal = $Ngood + $Ngone + $Ndead + $Nsing + $Ndups + $Nsbck + $Nxbck + $Ncul1 + $Ncul2 + $Ncul3 + $Nfail; +if ($Ntotal != $Nchip) { die "ERROR: inconsistent count of chips\n"; } + +# update the stats in the database +my $sql_update_stats = "UPDATE nights set"; +$sql_update_stats .= " Nchip = $Nchip"; +$sql_update_stats .= ", Ngood = $Ngood"; +$sql_update_stats .= ", Ndead = $Ndead"; +$sql_update_stats .= ", Ngone = $Ngone"; +$sql_update_stats .= ", Nsing = $Nsing"; +$sql_update_stats .= ", Ndups = $Ndups"; +$sql_update_stats .= ", Nsbck = $Nsbck"; +$sql_update_stats .= ", Nxbck = $Nxbck"; +$sql_update_stats .= ", Ncul1 = $Ncul1"; +$sql_update_stats .= ", Ncul2 = $Ncul2"; +$sql_update_stats .= ", Ncul3 = $Ncul3"; +$sql_update_stats .= ", Nfail = $Nfail"; +$sql_update_stats .= " WHERE dateobs = $dateword"; + +my $query = $raw_dbh->prepare($sql_update_stats); +$query->execute () or die "invalid response from RAW database\n"; + +exit 0; + +# read the list of chip_ids: +sub load_chip_list { + my $chiplist = $_[0]; + + my @chip_ids; + + open(CHIPS,"$topdir/$chiplist") or die "cannot open file $chiplist\n"; + + my @lines = ; + + foreach my $line (@lines) { + chomp $line; + if ($line =~ /^\s*#/) { next; } # skip lines with leading # + my @words = split (/\s+/,$line); + + if (@words != 1) { die "error in chip list $line\n"; } + push (@chip_ids, $words[0]); + } + close (CHIPS); + + return @chip_ids; +} + +sub parse_line { + + my $line = $_[0]; + my $name = $_[1]; + + my @words = split (/\s+/,$line); + + unless ($name eq $words[0]) { die "unexpected entry $line\n"; } + + return $words[1]; +} + +sub parse_cmdopts { + + my ($dateobs, $chiplist, $stage, $topdir); + GetOptions( 'dateobs=s' => \$dateobs, 'chiplist=s' => \$chiplist, 'topdir=s' => \$topdir, 'stage=s' => \$stage) || pod2usage(2); + + pod2usage( -msg => "Cannot determine target date, use --dateobs YYYY/MM/DD", -exitval => 2) unless defined $dateobs; + pod2usage( -msg => "Define the stage (0, 1, etc) with --stage", -exitval => 2) unless defined $stage; + pod2usage( -msg => "Specify the top-level output directory with --topdir", -exitval => 2) unless defined $topdir; + pod2usage( -msg => "Provide a list of chips with --chiplist", -exitval => 2) unless defined $chiplist; + + # check that stage is an integer + unless (is_integer ($stage)) { die "stage must be a non-negative integer\n"; } + unless ($stage >= 0) { die "stage must be a non-negative integer\n"; } + + # check the date format (require YYYY/MM/DD and use that from 00:00:00 to 23:59:59 or take start and end?) + + my ($date_year, $date_month, $date_day) = split "/",$dateobs; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) unless defined $date_year and defined $date_month and defined $date_day; + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_year < 2008) or ($date_year > 2030); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_month < 1) or ($date_month > 12); + pod2usage( -msg => "Cannot interpret date, use YYYY/MM/DD", -exitval => 2) if ($date_day < 1) or ($date_day > 31); + + my $fulldate = "$date_year/$date_month/$date_day"; + my $dateword = sprintf ("%4d%02d%02d", $date_year, $date_month, $date_day); + + return ($fulldate, $dateword, $chiplist, $stage, $topdir); +} + +# parse the nebulous.config file: +sub parse_db_config { + + my $conffile = $_[0]; + + my ( $rawdb_name, $rawdb_host, $rawdb_port, $rawdb_pass, $rawdb_user ); + + if (-e $conffile) { + open(IN,$conffile); + while() { + chomp; + if ($_ =~ /^\s*#/) { next; } # skip lines with leading # + if ($_ =~ /RAW_NAME /) { $rawdb_name = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_HOST /) { $rawdb_host = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_USER /) { $rawdb_user = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PASS /) { $rawdb_pass = (split /\s+/,$_)[2]; } + if ($_ =~ /RAW_PORT /) { $rawdb_port = (split /\s+/,$_)[2]; } + } + close(IN); + print STDERR "raw dbh config: $rawdb_name $rawdb_host $rawdb_port $rawdb_user $rawdb_pass\n"; + } + + pod2usage( -msg => "Cannot configure raw database (dbname)", -exitval => 2) unless defined $rawdb_name; + pod2usage( -msg => "Cannot configure raw database (dbhost)", -exitval => 2) unless defined $rawdb_host; + pod2usage( -msg => "Cannot configure raw database (dbuser)", -exitval => 2) unless defined $rawdb_user; + pod2usage( -msg => "Cannot configure raw database (dbport)", -exitval => 2) unless defined $rawdb_port; + pod2usage( -msg => "Cannot configure raw database (dbpass)", -exitval => 2) unless defined $rawdb_pass; + + my $rawdb_dsn = "DBI:mysql:database=$rawdb_name:host=$rawdb_host"; + if ($rawdb_port ne "NONE") { $rawdb_dsn .= ":port=$rawdb_port"; } + my $raw_dbh = DBI->connect($rawdb_dsn, $rawdb_user, $rawdb_pass, { RaiseError => 1, AutoCommit => 1} ) or die "Unable to connect to database $DBI::errstr\n"; + + return ($raw_dbh); +} + +sub vsystem { + print STDERR "@_\n"; + # return 0; + my $status = system ("@_"); + return $status; +} + +sub is_integer { + defined $_[0] && $_[0] =~ /^[+-]?\d+$/; +} +