#!/usr/bin/perl
#
#  This is a perl script test harness which will recursively search the
#  directory tree looking for files named UnitTest and then execute
#  the script
#
#  SYNOSIS :  FullUnitTest options arguements
#
#       where  options =
#             --verbose     Display extra information to user
#             --noverbose   Don't display extra information to user
#             --recursiv e  Recursively run tests in directory tree
#             --norecursive Test only the specified or current directory
#             --silent      Don't display any information to user
#             --nosilent    Display progress of script to user
#
#              arguements = directory(ies) to perform tests
#
#  RETURN : integer number of tests which failed
#
#  $Revision: 1.2 $  $Name: not supported by cvs2svn $
#  $Date: 2004-05-06 00:45:10 $
#
#  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
#
##############################################################################

# Provide functions for determining the pathname of current working directory
use Cwd;

# Provides functions for handling long command line options
use Getopt::Long;

# Assign variables based on the presence of command line options to the script
# The ! option allows for --nooption to be set to zero
# (e.g. --noverbose --recursive causes $verbose=0 and $recursive=1)
GetOptions("verbose!"=>\$verbose,
           "recursive!"=>\$recursive,
           "silent!"=>\$silent);

# Check if both silent and verbose options are set and if so stop the script
die "Can't specify both verbose and silent options." if ($verbose && $silent);

# Set up the PSLIB_ROOT environment variable if the user doesn't have
if (! $ENV{'PSLIB_ROOT'})  {
    # Use the directory directly above where FullUnitTest script resides
    $PSLIB_ROOT = `cd ..;pwd`;
    # Remove newline for the end of path returned
    chomp($PSLIB_ROOT);
    # Set the environment variable
    $ENV{'PSLIB_ROOT'} = $PSLIB_ROOT;
    if ($verbose) {
        print("PSLIB_ROOT not found: set to $PSLIB_ROOT.\n");
    }
}

# add PSLIB_ROOT/lib to LD_LIBRARY_PATH and DYLD_LIBRARY_PATH environment
# variables
$ENV{'LD_LIBRARY_PATH'} = "$ENV{'PSLIB_ROOT'}/lib:$ENV{'LD_LIBRARY_PATH'}";
$ENV{'DYLD_LIBRARY_PATH'} = "$ENV{'PSLIB_ROOT'}/lib:$ENV{'DYLD_LIBRARY_PATH'}";

# Initialize variables for counting the makes and test failures and the
# total makes and tests performed
$makeFailCount = 0;
$testFailCount = 0;
$totalTests = 0;
$totalMakes = 0;

# Initialize variable indicating how many arguements were passed
$args = 0;

# Loop through all the arguements passed to the script
foreach (@ARGV) {
    # Remove newline if there is one
    chomp;
    # Increment number of arguements found
    $args++;
    # Set variable to current working directory
    $cwd = cwd;
    # Change directory to the directory of the arguement
    chdir($_);
    # Check for the recursive option
    if ($recursive) {
        # Invoke subroutine to go to the lowest directory in tree
        # starting at the specified directory
        &worm($_);
    } else {
        # Invoke subroutines to run the test at the specified directory
        &makeTestDrivers($_);
        &executeTestDrivers($_);
    }
    # Change directory back to the directory where FullUnitTest was invoked
    chdir($cwd);
}

# Check if there were no arguements specified
if ($args == 0){
    # Display message to user that all directories under the current will be
    # tested
    print("Recursively testing current directory tree.\n");
    # Invoke subroutine to go to recursively test each directory in tree
    &worm($ENV{"PWD"});
}

# Check if there were any failures during make or testing
if ($makeFailCount>0 || $testFailCount>0) {
    # Display summary of failures
    print("\nMake Failures = $makeFailCount out of $totalMakes");
    print("\nTest Failures = $testFailCount out of $totalTests\n");
    print("\nMakes that failed:\n  ".join("\n  ",@makesFailed) . "\n") if $makeFailCount;
    print("\nTests that failed:\n  ".join("\n  ",@testsFailed) . "\n") if $testFailCount;
} else {
    # Display message of all makes and tests pass to user if silent option
    # not specified
    print("\nAll $totalTests Tests Passed.\n") if (! $silent);
}

# Exit with the number of tests that failed
exit($testFailCount);


################################################################################
#
# SUBROUTINE: worm
#
#     Description:  This subroutine will perform the necessary unit tests be
#                   calling makeTestDrivers and executeTestDrivers  subroutines
#                   and then check each subdirectory below the base directory 
#                   recursively.
#
#     Parameter(s):  base directory to start testing
#
#     Return:  None
#
################################################################################

sub worm {
    # Assign local variable to input parameter
    local($base_dir) = @_;
    local(@files, $i);

    # Invoke subroutine to make test driver in the base directory
    &makeTestDrivers($base_dir);

    # Invoke subroutine to execute tests in the base directory
    &executeTestDrivers($base_dir);

    # Create array of entries found in directory
    @files = <*>;

    # Loop through the file list looking for another directory that is not
    # labelled CVS and recursively invoke subroutine worm
    $i = 0;
    while ($files[$i]) {
        # Check for directory and directory not labelled CVS
        if (-d $files[$i] && ($files[$i] ne "CVS") && ($files[$i] ne "temp")
                          && ($files[$i] ne "verified")) {
            # Change current directory to directory found
            chdir($files[$i]);
            # Invoke subroutine worm again
            &worm($files[$i]);
            # Change current directory back to parent
            chdir("..");
        }
        # Increment file list index
        $i++;
    }
}

################################################################################
#
#  SUBROUTINE: makeTestDrivers
#
#      Description:  This subroutine will perform a make for all the necessary
#                    test drivers.    This will occur in the specified
#                    base directory which is passed as the only parameter.
#
#      Parameter(s):  base directory where make and test drivers are located
#
#      Return:  None
#
################################################################################

sub makeTestDrivers{
    local($base_dir) = @_;
    local($pwd);

    # Set variable pwd to current working directory
    $pwd = cwd;

    # Display message to user in which directory testing is taken place only
    # if the silent command option was not specified
    print("---- Testing: $pwd ----\n") if ( ! $silent);

    # Check for the existence of a file name Makefile in current directory
    if (-e "Makefile") {

        # Increment the total number of makes executed
        $totalMakes++;

        # Execute the make clean
        `make clean`;

        # Execute the make distclean
        `make distclean`;

        # Execute the make and save results
        $_ = join("\n|| ",split("\n", "\n" . `make`) );

        # Check the output of make for return value != 0 or any of the
        # following words: FAILED, FAULT, ERROR, Not found, SIGNAL
        if (($? != 0) || m/FAILED/i || m/FAULT/i || m/\bERROR/i
                      || m/Not found/i || m/SIGNAL/i) {
            # Display the errored output of make if silent option not enabled
            print("$_\n") if ( ! $silent );
            # Display the make failed in the current directory
            print("\nMake for $pwd Failed\n");
            # Increment the number of makes that have failed
            $makeFailCount++;
            # Push the current working directory onto the list of failed
            # make directories list
            push(@makesFailed, $pwd);
        } else {
            # Display the results of the successful make if verbose option set
            print("$_\n") if $verbose;
            # Display the make was successful if silent option not set
            print("\nMake successful.\n") if ( ! $silent);
            `make install`;
        }
    }
}

################################################################################
#
#  SUBROUTINE: executeTestDrivers
#
#      Description:  This subroutine will execute all the necessary
#                    test drivers.    This will occur in the specified
#                    base directory which is passed as the only parameter.
#
#      Parameter(s):  base directory where make and test drivers are located
#
################################################################################

sub executeTestDrivers{
    local($base_dir) = @_;
    local(@files,$j);
    local($pwd);
    local($exitValue) = 0;

    # Set variable to pwd to current test directory
    $pwd = cwd;

    # Create a list of all elements in base directory
    @files = <*>;

    # Loop through all the items in the files array
    $initialTest = 0;
    $j = 0;
    while ( $files[$j]) {
        # Check that the item is not a directory and is executable and
        # conforms to the test driver naming convention TST
        if ( ! (-d $files[$j]) && (-x $files[$j]) && 
             ( $files[$j] =~ /^TST/i || $files[$j] =~ /^ATST/i ) ) {

            # Increment total number of test drivers executed
            $totalTests++;

            # Perform subdirectory checks only the first time a test
            # file is found with the list of enteries
            if ( $initialTest == 0 ) {
                $initialTest++;

                # Check for a temp subdirectory already exists and if not
                # then create one
                &checkForTempDirectory();

                # Check for the presence of verified directory
                if ( &checkForVerifiedDirectory() ) {
            
                    # Display message to user that verified directories do not 
                    # exist eventhough TST files have been found
#                    print("Unable to execute test drivers in $base_dir\n");
                    print("No verified subdirectory present.\n");
                    # Increment test fail count
 #                   $testFailCount++;
                    # Save the file failed
#                    push(@testsFailed, $pwd . "/" . $files[$j]);
#                    return;
                }
            }

            # Display message to user of which test driver is being executed
            print("--- Executing test driver $files[$j]\n") if ( ! $silent );
            # Execute the test driver
            system("./$files[$j] 1> temp/$files[$j].stdout 2> temp/$files[$j].stderr");
            # Check result of test driver
            if ( ( ($? != 0) && ($files[$j] =~ /^TST/i) ) ||
                 ( ($? == 0) && ($files[$j] =~ /^ATST/i ) ) ) {
                # Display test driver failed
                print("Test driver: $files[$j] Failed (Return value $?).\n");
                # Increment the total number of test failed
                $testFailCount++;
                # Push the current working directory on the list of failed
                push(@testsFailed, $pwd . "/" . $files[$j]);
            } else {
                # Create filter versions of STDOUT and STDERR to replace variable
                # items such as date, time and host
                &filterStdFiles($files[$j]);
                # Perform difference on STDOUT file collected
                # with verified files
                $exitValue = &compareStream("verified/$files[$j].stdout");
                if ( $exitValue & 2 ) {
                    # STDOUT verified doesn't exist. Search STDOUT capture
                    # for strings indicating error or failure
                    $exitValue |= &errorStrSearch("$files[$j].stdout");
                }
                if ( ($exitValue & 8) || ($exitValue & 64) ) {
                    # Increment the total number of test failed
                    $testFailCount++;
                    # Push test on failed list
                    push(@testsFailed, $pwd . "/" . $files[$j]);
                } else {
                    # Perform difference on STDERR file collection with verified
                    $exitValue = &compareStream("verified/$files[$j].stderr");
                    if ( $exitValue & 4 ) {
                        # STDERR verified doesn't exist. Search STDERR capture
                        # for strings indicating error or failure
                        $exitValue |= &errorStrSearch("$files[$j].stderr");
                    }
                    if ( ($exitValue & 16) || ($exitValue & 128) ) {
                        # Increment the total number of tests failed
                        $testFailCount++;
                        # Push test on failed list
                        push(@testsFailed, $pwd . "/" . $files[$j]);
                    } else {
                       print("Test successful\n") if ( ! $silent);
                    }
                }
            }
        }
        $j++;
    }
}

################################################################################
#
#  SUBROUTINE: errorStrSearch
#
#      Description:  This subroutine will search the file specified in its
#                    parameter for error strings and if they do exists
#                    the appropriate value will be returned.
#
#      Parameter(s): fileName - input file to search for strings
#
#      Return:   0  -  No error strings found
#               64  -  Error strings found in STDOUT
#              128  -  Error strings found in STDERR
#
###############################################################################

sub errorStrSearch {
    local($fileName) = @_;
    local($returnVal) = 0;

    # Open the captured file
    open(CAPT_FILE, "<temp/$fileName");

    # Scan through all the lines of the file searching for error strings
    while ( <CAPT_FILE> ) {
        if ( m/FAIL/i || m/FAULT/i || m/ERROR/i || m/Not Found/i
                      || m/SIGNAL/i || m/NO SUCH FILE/i || m/|E|/i
                      || m/|A|/i ) {
            if ( $fileName =~ m/out/ ) {
                print("        Failed - File $fileName contains error strings.\n");
                $returnVal = 64;
            } elsif ( $fileName =~ m/err/ ) {
                print("        Failed - File $fileName contains error strings.\n");
                $returnVal = 128;
            }
            last;
        }
    }
    # Close the capture file
    close(CAPT_FILE);
    # Return the return value
    return($returnVal);
}

################################################################################
#
#  SUBROUTINE: compareStream
#
#      Description:  This subroutine will compare the captured stream file with
#                    a file in the verified directory if one exists.
#
#      Parameter(s): streamFile - input file to compare
#
#      Return:   0  -  Compare successful
#                2  -  STDOUT verified file doesn't exist
#                4  -  STDERR verified file doesn't exist
#                8  -  STDOUT verified file doesn't compare
#               16  -  STDERR verified file doesn't compare
#
###############################################################################

sub compareStream {
    local($streamFile) = @_;
    local($returnVal) = 0;
    local($tempFile) = "";

    # Check for existence of verified STD stream files
    if ( !( -e $streamFile) ) {

        # Display message to user that verified STDOUT file doesn't exist
        print("        File $streamFile doesn't exist.\n");

        # Set exit value bit 1 to indicate proper failure
        if ( $streamFile =~ /out$/ ) {
            $returnVal |= 2;
        } elsif ( $streamFile =~ /err$/ ) {
            $returnVal |= 4;
        }
    } else {

        # Verified STD stream file exists

        # Create name of the temp file to compare
        $tempFile = $streamFile;
        $tempFile =~ s/verified/temp/;
        $tempFile = $tempFile . ".mod";

        # Perform difference on the STD stream files
        $diffstdout = `diff $streamFile $tempFile`;

        # Check the return value of the difference
        if ( $? != 0 ) {

            # Difference of STD stream files failed

            # Check for STDOUT in file name to convey appropirate return value
            if ( $streamFile =~ /out$/ ) {
               # Display message of the failure of difference to user
               print("        Failed - STDOUT difference\n$diffstdout\n");

               # Exit value to indicate STDOUT did not compare
               $returnVal |= 8;
            } elsif ( $streamFile =~ /err$/ ) {
               # Display message of the failure of difference to user
               print("        Failed - STDERR difference\n$diffstdout\n");

               # Exit value to indicate STDERR did not compare
               $returnVal |= 16;
            }
        }
    }

    # Return the result of the compare
    return($returnVal);
}


################################################################################
#
#  SUBROUTINE: checkForTempDirectory
#
#      Description:  This subroutine will check for the existence of a temp
#                    directory to store STDOUT, STDERR files during test.  If
#                    the subdirectory doesn't exist, it will be created.
#
#      Parameter(s):  None
#
#      Return: None
#
################################################################################

sub checkForTempDirectory{

    # Check if a temp directory already exists in the current work directory
    if ( !( -e "temp" )) {
        # Create temp directory to store STDOUT, STDERR files during test
        `mkdir temp`;
        # Display message of new directory created
        print("Creating temp directory\n") if ( $verbose );
    }
}

################################################################################
#
#  SUBROUTINE: checkForVerifiedDirectory
#
#      Description:  This subroutine will check for the existence of a verified
#                    directory which stores verified STDOUT, STDERR files for
#                    comparison during the test.
#
#      Parameter(s):  None
#
#      Return: 0 - subdirectory present
#              1 - subdirectory not present
#
################################################################################

sub checkForVerifiedDirectory{
    $returnValue = 0;
    # Check if verified subdirectory exists in the current working directory
    if ( !( -e "verified")) {
       # Set return value to 1
       $returnValue = 1;
    }
    return($returnValue);
}

################################################################################
#
#  SUBROUTINE: filterStdFiles
#
#      Description:  This subroutine will filter variable items in the STDOUT
#                    and STDERR files captured.  Date, time and host names
#                    are variable items in the log messages captured.
#
#      Parameter(s):  test file name
#
#      Return: None
#
################################################################################

sub filterStdFiles{

   local($fileName) = @_;

   # Open the STDOUT file for reading
   open(OUTFILE, "< temp/$fileName.stdout");
   # Open mod file to place filtered STDOUT
   open(MODFILE, "> temp/$fileName.stdout.mod");
   # Replace the variable data, time and host information with constants
   while( <OUTFILE> ) {
       s/\d+:\d+:\d+/ <DATE> /;
       s/\s+\d+:\d+:\d+\w/ <TIME> /;
       s/\|[\w\.]+\|/<HOST> |/;
       print MODFILE ($_);
   }
   # Close mod file
   close(MODFILE);
   # Close STDERR file
   close(OUTFILE);

   # Open the STDERR file for reading
   open(OUTFILE, "< temp/$fileName.stderr");
   # Open mod file to place filtered STDERR
   open(MODFILE, "> temp/$fileName.stderr.mod");
   # Replace the variable date, time and host information with constants
   while( <OUTFILE> ) {
       s/\d+:\d+:\d+/ <DATE> /;
       s/\s+\d+:\d+:\d+\w/ <TIME> /;
       s/\|[\w\.]+\|/<HOST> |/;
       print MODFILE ($_);
   }
   # Close mod file
   close(MODFILE);
   # Close STDERR file
   close(OUTFILE);
}

