Index: trunk/psModules/test/.cvsignore =================================================================== --- trunk/psModules/test/.cvsignore (revision 2139) +++ trunk/psModules/test/.cvsignore (revision 2140) @@ -6,2 +6,3 @@ tst_pmNonLinear tst_pmSubtractBias +temp Index: trunk/psModules/test/FullUnitTest =================================================================== --- trunk/psModules/test/FullUnitTest (revision 2140) +++ trunk/psModules/test/FullUnitTest (revision 2140) @@ -0,0 +1,710 @@ +#!/usr/bin/perl +# +# This is a perl script test harness which will recursively search the +# directory tree looking for files named UnitTest and then execute +# the script +# +# SYNOSIS : FullUnitTest options arguements +# +# where options = +# --verbose Display extra information to user +# --noverbose Don't display extra information to user +# --recursiv e Recursively run tests in directory tree +# --norecursive Test only the specified or current directory +# --silent Don't display any information to user +# --nosilent Display progress of script to user +# +# arguements = directory(ies) to perform tests +# +# RETURN : integer number of tests which failed +# +# $Revision: 1.1 $ $Name: not supported by cvs2svn $ +# $Date: 2004-10-14 23:58:56 $ +# +# Copyright 2004 Maui High Performance Computing Center, University of Hawaii +# +############################################################################## + +# Provide functions for determining the pathname of current working directory +use Cwd; + +# Provides functions for handling long command line options +use Getopt::Long; + +# Assign variables based on the presence of command line options to the script +# The ! option allows for --nooption to be set to zero +# (e.g. --noverbose --recursive causes $verbose=0 and $recursive=1) +GetOptions( + "verbose!" => \$verbose, + "recursive!" => \$recursive, + "silent!" => \$silent, + "clean!" => \$clean +); + +# Check if both silent and verbose options are set and if so stop the script +die "Can't specify both verbose and silent options." if ( $verbose && $silent ); + +# Set up the PSLIB_ROOT environment variable if the user doesn't have +if ( !$ENV{'PSLIB_ROOT'} ) { + + # Use the directory directly above where FullUnitTest script resides + $PSLIB_ROOT = `cd ..;pwd`; + + # Remove newline for the end of path returned + chomp($PSLIB_ROOT); + + # Set the environment variable + $ENV{'PSLIB_ROOT'} = $PSLIB_ROOT; + print("PSLIB_ROOT not found: set to $PSLIB_ROOT.\n") if $verbose; +} + +# add PSLIB_ROOT/lib to LD_LIBRARY_PATH and DYLD_LIBRARY_PATH environment +# variables +$ENV{'LD_LIBRARY_PATH'} = "$ENV{'PSLIB_ROOT'}/lib:$ENV{'LD_LIBRARY_PATH'}"; +$ENV{'DYLD_LIBRARY_PATH'} = "$ENV{'PSLIB_ROOT'}/lib:$ENV{'DYLD_LIBRARY_PATH'}"; + +# Initialize variables for counting the makes and test failures and the +# total makes and tests performed +$makeFailCount = 0; +$testpointFailCount = 0; +$testDriverFailCount = 0; +$totalTestpoints = 0; +$totalTestDrivers = 0; +$totalMakes = 0; + +# Initialize variable indicating how many arguements were passed +$args = 0; + +# Loop through all the arguements passed to the script +foreach (@ARGV) { + + # Remove newline if there is one + chomp; + + # Increment number of arguements found + $args++; + + # Set variable to current working directory + $cwd = cwd; + + # Change directory to the directory of the arguement + chdir($_); + + # Check for the recursive option + if ($recursive) { + + # Invoke subroutine to go to the lowest directory in tree + # starting at the specified directory + &worm($_); + } + else { + + # Invoke subroutines to run the test at the specified directory + &makeTestDrivers($_); + &executeTestDrivers($_); + } + + # Change directory back to the directory where FullUnitTest was invoked + chdir($cwd); +} + +# Check if there were no arguements specified +if ( $args == 0 ) { + + # Display message to user that all directories under the current will be + # tested + print("Recursively testing current directory tree.\n") if $verbose; + + # Invoke subroutine to go to recursively test each directory in tree + &worm( $ENV{"PWD"} ); +} + +# Check if there were any failures during make or testing +if ( $makeFailCount > 0 || $testDriverFailCount > 0 ) { + + # Display summary of failures + print("\nMake Failures = $makeFailCount out of $totalMakes"); + print( +"\nTest Driver Failures = $testDriverFailCount out of $totalTestDrivers\n" + ); + print( "\nMakes that failed:\n " . join( "\n ", @makesFailed ) . "\n" ) + if $makeFailCount; + print( "\nTests that failed:\n " . join( "\n ", @testsFailed ) . "\n" ) + if $testDriverFailCount; +} +else { + + # Display message of all makes and tests pass to user if silent option + # not specified + print( +"\nAll $totalTestDrivers Test Drivers Passed with $totalTestpoints Testpoints.\n" + ) + if ( !$silent ); +} + +# Exit with the number of tests that failed +exit($testDriverFailCount); + +################################################################################ +# +# SUBROUTINE: worm +# +# Description: This subroutine will perform the necessary unit tests be +# calling makeTestDrivers and executeTestDrivers subroutines +# and then check each subdirectory below the base directory +# recursively. +# +# Parameter(s): base directory to start testing +# +# Return: None +# +################################################################################ + +sub worm { + + # Assign local variable to input parameter + local ($base_dir) = @_; + local ( @files, $i ); + + # Invoke subroutine to make test driver in the base directory + &makeTestDrivers($base_dir); + + # Invoke subroutine to execute tests in the base directory + &executeTestDrivers($base_dir); + + # Create array of entries found in directory + @files = <*>; + + # Loop through the file list looking for another directory that is not + # labelled CVS and recursively invoke subroutine worm + $i = 0; + while ( $files[$i] ) { + + # Check for directory and directory not labelled CVS + if ( -d $files[$i] + && ( $files[$i] ne "CVS" ) + && ( $files[$i] ne "temp" ) + && ( $files[$i] ne "verified" ) ) + { + + # Change current directory to directory found + chdir( $files[$i] ); + + # Invoke subroutine worm again + &worm( $files[$i] ); + + # Change current directory back to parent + chdir(".."); + } + + # Increment file list index + $i++; + } +} + +################################################################################ +# +# SUBROUTINE: makeTestDrivers +# +# Description: This subroutine will perform a make for all the necessary +# test drivers. This will occur in the specified +# base directory which is passed as the only parameter. +# +# Parameter(s): base directory where make and test drivers are located +# +# Return: None +# +################################################################################ + +sub makeTestDrivers { + local ($base_dir) = @_; + local ($pwd); + + # Set variable pwd to current working directory + $pwd = cwd; + + # Display message to user in which directory testing is taken place only + # if the silent command option was not specified + print("---- Entering $pwd ----\n") if ( !$silent ); + + # Check for the existence of a file name Makefile in current directory + if ( -e "Makefile" ) { + + # Increment the total number of makes executed + $totalMakes++; + + if ($clean) { + + # Execute the make clean + `make clean`; + + # Execute the make distclean + `make distclean`; + } + + # Execute the make and save results + $_ = join( "\n|| ", split( "\n", "\n" . `make` ) ); + + # Check the output of make for return value != 0 or any of the + # following words: FAILED, FAULT, ERROR, Not found, SIGNAL + if ( ( $? != 0 ) + || m/FAILED/i + || m/FAULT/i + || m/\bERROR/i + || m/Not found/i + || m/SIGNAL/i ) + { + + # Display the errored output of make if silent option not enabled + print("$_\n") if ( !$silent ); + + # Display the make failed in the current directory + print("\nMake for $pwd Failed\n"); + + # Increment the number of makes that have failed + $makeFailCount++; + + # Push the current working directory onto the list of failed + # make directories list + push( @makesFailed, $pwd ); + } + else { + + # Display the results of the successful make if verbose option set + print("$_\n") if $verbose; + + # Display the make was successful if silent option not set + print("\nMake successful.\n") if ( !$silent ); + if ($clean) { + `make install`; + } + } + } +} + +################################################################################ +# +# SUBROUTINE: executeTestDrivers +# +# Description: This subroutine will execute all the necessary +# test drivers. This will occur in the specified +# base directory which is passed as the only parameter. +# +# Parameter(s): base directory where make and test drivers are located +# +################################################################################ + +sub executeTestDrivers { + local ($base_dir) = @_; + local ( @files, $j ); + local ($pwd); + local ($exitValue) = 0; + + # Set variable to pwd to current test directory + $pwd = cwd; + + # Create a list of all elements in base directory + @files = <*>; + + # Loop through all the items in the files array + $initialTest = 0; + $j = 0; + while ( $files[$j] ) { + + # Check that the item is not a directory and is executable and + # conforms to the test driver naming convention TST + if ( !( -d $files[$j] ) + && ( -x $files[$j] ) + && ( $files[$j] =~ /^TST/i || $files[$j] =~ /^ATST/i ) ) + { + + # Increment total number of test drivers executed + $totalTestDrivers++; + + # Perform subdirectory checks only the first time a test + # file is found with the list of enteries + if ( $initialTest == 0 ) { + $initialTest++; + + # Check for a temp subdirectory already exists and if not + # then create one + &checkForTempDirectory(); + + # Check for the presence of verified directory + if ( &checkForVerifiedDirectory() ) { + + # Display message to user that verified directories do not + # exist eventhough TST files have been found + # print("Unable to execute test drivers in $base_dir\n"); + print("No verified subdirectory present.\n") if ($verbose); + + # Increment test fail count + # $testDriverFailCount++; + # Save the file failed + # push(@testsFailed, $pwd . "/" . $files[$j]); + # return; + } + } + + # Display message to user of which test driver is being executed + print("--- Executing test driver $files[$j]\n") if ( !$silent ); + + # Execute the test driver + system( +"./$files[$j] 1> temp/$files[$j].stdout 2> temp/$files[$j].stderr" + ); + $retVal = $?; + + # Count testpoints + $testPattern = "\"\\*\\*\\*\\* TESTPOINT \\*\\*\\*\\*\""; + $totalPoints = `grep -c $testPattern temp/$files[$j].stdout`; + $totalPoints += `grep -c $testPattern temp/$files[$j].stderr`; + $failPoints = + `grep "> TESTPOINT FAILED" temp/$files[$j].stdout | wc -l`; + $failPoints += + `grep "> TESTPOINT FAILED" temp/$files[$j].stderr | wc -l`; + $testpointFailCount += $failPoints; + $totalTestpoints += $totalPoints; + + # Check result of test driver + if ( ( ( $retVal != 0 ) && ( $files[$j] =~ /^TST/i ) ) + || ( ( $retVal == 0 ) && ( $files[$j] =~ /^ATST/i ) ) ) + { + + # Display test driver failed + $failPoints++; + print( + "Test driver: $files[$j] Failed (Return value $retVal).\n"); + + # Increment the total number of test failed + $testDriverFailCount++; + + # Push the current working directory on the list of failed + push( @testsFailed, $pwd . "/" . $files[$j] ); + } + else { + + # Create filter versions of STDOUT and STDERR to replace variable + # items such as date, time and host + &filterStdFiles( $files[$j] ); + + # Perform difference on STDOUT file collected + # with verified files + $exitValue = &compareStream("verified/$files[$j].stdout"); + if ( $exitValue & 2 ) { + + # STDOUT verified doesn't exist. Search STDOUT capture + # for strings indicating error or failure + $exitValue |= &errorStrSearch("$files[$j].stdout"); + } + if ( ( $exitValue & 8 ) || ( $exitValue & 64 ) ) { + + # Increment the total number of test failed + $testDriverFailCount++; + + # Push test on failed list + print( +"Test failed ($failPoints out of $totalPoints testpoints failed)\n" + ) + if ( !$silent ); + push( @testsFailed, $pwd . "/" . $files[$j] ); + } + else { + + # Perform difference on STDERR file collection with verified + $exitValue = &compareStream("verified/$files[$j].stderr"); + if ( $exitValue & 4 ) { + + # STDERR verified doesn't exist. Search STDERR capture + # for strings indicating error or failure + $exitValue |= &errorStrSearch("$files[$j].stderr"); + } + if ( ( $exitValue & 16 ) || ( $exitValue & 128 ) ) { + + # Increment the total number of tests failed + $testDriverFailCount++; + + # Push test on failed list + print( +"Test failed ($failPoints out of $totalPoints testpoints failed)\n" + ) + if ( !$silent ); + push( @testsFailed, $pwd . "/" . $files[$j] ); + } + else { + print("Test successful ($totalPoints testpoints)\n") + if ( !$silent ); + } + } + } + } + $j++; + } +} + +################################################################################ +# +# SUBROUTINE: errorStrSearch +# +# Description: This subroutine will search the file specified in its +# parameter for error strings and if they do exists +# the appropriate value will be returned. +# +# Parameter(s): fileName - input file to search for strings +# +# Return: 0 - No error strings found +# 64 - Error strings found in STDOUT +# 128 - Error strings found in STDERR +# +############################################################################### + +sub errorStrSearch { + local ($fileName) = @_; + local ($returnVal) = 0; + + # Open the captured file + open( CAPT_FILE, ") { + if ( m/FAIL/i + || m/FAULT/i + || m/ERROR/i + || m/Not Found/i + || m/SIGNAL/i + || m/NO SUCH FILE/i + || m/\|E\|/i + || m/\|A\|/i ) + { + print; + if ( $fileName =~ m/out/ ) { + print( + " Failed - File $fileName contains error strings.\n" + ); + $returnVal = 64; + } + elsif ( $fileName =~ m/err/ ) { + print( + " Failed - File $fileName contains error strings.\n" + ); + $returnVal = 128; + } + last; + } + } + + # Close the capture file + close(CAPT_FILE); + + # Return the return value + return ($returnVal); +} + +################################################################################ +# +# SUBROUTINE: compareStream +# +# Description: This subroutine will compare the captured stream file with +# a file in the verified directory if one exists. +# +# Parameter(s): streamFile - input file to compare +# +# Return: 0 - Compare successful +# 2 - STDOUT verified file doesn't exist +# 4 - STDERR verified file doesn't exist +# 8 - STDOUT verified file doesn't compare +# 16 - STDERR verified file doesn't compare +# +############################################################################### + +sub compareStream { + local ($streamFile) = @_; + local ($returnVal) = 0; + local ($tempFile) = ""; + + # Check for existence of verified STD stream files + if ( !( -e $streamFile ) ) { + + # Display message to user that verified STDOUT file doesn't exist + print(" File $streamFile doesn't exist.\n") if ($verbose); + + # Set exit value bit 1 to indicate proper failure + if ( $streamFile =~ /out$/ ) { + $returnVal |= 2; + } + elsif ( $streamFile =~ /err$/ ) { + $returnVal |= 4; + } + } + else { + + # Verified STD stream file exists + + # Create name of the temp file to compare + $tempFile = $streamFile; + $tempFile =~ s/verified/temp/; + $tempFile = $tempFile . ".mod"; + + # Perform difference on the STD stream files + $diffstdout = `diff $streamFile $tempFile`; + + # Check the return value of the difference + if ( $? != 0 ) { + + # Difference of STD stream files failed + + # Check for STDOUT in file name to convey appropirate return value + if ( $streamFile =~ /out$/ ) { + + # Display message of the failure of difference to user + print(" Failed - STDOUT difference\n$diffstdout\n"); + + # Exit value to indicate STDOUT did not compare + $returnVal |= 8; + } + elsif ( $streamFile =~ /err$/ ) { + + # Display message of the failure of difference to user + print(" Failed - STDERR difference\n$diffstdout\n"); + + # Exit value to indicate STDERR did not compare + $returnVal |= 16; + } + } + } + + # Return the result of the compare + return ($returnVal); +} + +################################################################################ +# +# SUBROUTINE: checkForTempDirectory +# +# Description: This subroutine will check for the existence of a temp +# directory to store STDOUT, STDERR files during test. If +# the subdirectory doesn't exist, it will be created. +# +# Parameter(s): None +# +# Return: None +# +################################################################################ + +sub checkForTempDirectory { + + # Check if a temp directory already exists in the current work directory + if ( !( -e "temp" ) ) { + + # Create temp directory to store STDOUT, STDERR files during test + `mkdir temp`; + + # Display message of new directory created + print("Creating temp directory\n") if ($verbose); + } +} + +################################################################################ +# +# SUBROUTINE: checkForVerifiedDirectory +# +# Description: This subroutine will check for the existence of a verified +# directory which stores verified STDOUT, STDERR files for +# comparison during the test. +# +# Parameter(s): None +# +# Return: 0 - subdirectory present +# 1 - subdirectory not present +# +################################################################################ + +sub checkForVerifiedDirectory { + $returnValue = 0; + + # Check if verified subdirectory exists in the current working directory + if ( !( -e "verified" ) ) { + + # Set return value to 1 + $returnValue = 1; + } + return ($returnValue); +} + +################################################################################ +# +# SUBROUTINE: filterStdFiles +# +# Description: This subroutine will filter variable items in the STDOUT +# and STDERR files captured. Date, time and host names +# are variable items in the log messages captured. +# +# Parameter(s): test file name +# +# Return: None +# +################################################################################ + +sub filterStdFiles { + + local ($fileName) = @_; + + # Open the STDOUT file for reading + open( OUTFILE, "< temp/$fileName.stdout" ); + + # Open mod file to place filtered STDOUT + open( MODFILE, "> temp/$fileName.stdout.mod" ); + + # Replace the variable data, time and host information with constants + $hostname = `hostname`; + chop $hostname; + while () { + s/\s+\d+:\d+:\d+\w/