#!/usr/bin/perl
#
#  This is a perl script test harness which will recursively search the
#  directory tree looking for files named UnitTest and then execute
#  the script
#
#  SYNOSIS :  FullUnitTest options arguements
#
#       where  options =
#             --verbose     Display extra information to user
#             --noverbose   Don't display extra information to user
#             --recursiv e  Recursively run tests in directory tree
#             --norecursive Test only the specified or current directory
#             --silent      Don't display any information to user
#             --nosilent    Display progress of script to user
#
#              arguements = directory(ies) to perform tests
#
#  RETURN : integer number of tests which failed
#
#  $Revision: 1.15 $  $Name: not supported by cvs2svn $
#  $Date: 2004-10-27 00:57:31 $
#
#  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
#
##############################################################################

# Provide functions for determining the pathname of current working directory
use Cwd;

# Provides functions for handling psS64 command line options
use Getopt::Long;

# Assign variables based on the presence of command line options to the script
# The ! option allows for --nooption to be set to zero
# (e.g. --noverbose --recursive causes $verbose=0 and $recursive=1)
GetOptions(
    "verbose!"   => \$verbose,
    "recursive!" => \$recursive,
    "silent!"    => \$silent,
    "clean!"     => \$clean
);

# Check if both silent and verbose options are set and if so stop the script
die "Can't specify both verbose and silent options." if ( $verbose && $silent );

# Set up the PSLIB_ROOT environment variable if the user doesn't have
if ( !$ENV{'PSLIB_ROOT'} ) {

    # Use the directory directly above where FullUnitTest script resides
    $PSLIB_ROOT = `cd ..;pwd`;

    # Remove newline for the end of path returned
    chomp($PSLIB_ROOT);

    # Set the environment variable
    $ENV{'PSLIB_ROOT'} = $PSLIB_ROOT;
    print("PSLIB_ROOT not found: set to $PSLIB_ROOT.\n") if $verbose;
}

# add PSLIB_ROOT/lib to LD_LIBRARY_PATH and DYLD_LIBRARY_PATH environment
# variables
$ENV{'LD_LIBRARY_PATH'}   = "$ENV{'PSLIB_ROOT'}/lib:$ENV{'LD_LIBRARY_PATH'}";
$ENV{'DYLD_LIBRARY_PATH'} = "$ENV{'PSLIB_ROOT'}/lib:$ENV{'DYLD_LIBRARY_PATH'}";

# Initialize variables for counting the makes and test failures and the
# total makes and tests performed
$makeFailCount       = 0;
$testpointFailCount  = 0;
$testDriverFailCount = 0;
$totalTestpoints     = 0;
$totalTestDrivers    = 0;
$totalMakes          = 0;

# Initialize variable indicating how many arguements were passed
$args = 0;

# Loop through all the arguements passed to the script
foreach (@ARGV) {

    # Remove newline if there is one
    chomp;

    # Increment number of arguements found
    $args++;

    # Set variable to current working directory
    $cwd = cwd;

    # Change directory to the directory of the arguement
    chdir($_);

    # Check for the recursive option
    if ($recursive) {

        # Invoke subroutine to go to the lowest directory in tree
        # starting at the specified directory
        &worm($_);
    }
    else {

        # Invoke subroutines to run the test at the specified directory
        &makeTestDrivers($_);
        &executeTestDrivers($_);
    }

    # Change directory back to the directory where FullUnitTest was invoked
    chdir($cwd);
}

# Check if there were no arguements specified
if ( $args == 0 ) {

    # Display message to user that all directories under the current will be
    # tested
    print("Recursively testing current directory tree.\n") if $verbose;

    # Invoke subroutine to go to recursively test each directory in tree
    &worm( $ENV{"PWD"} );
}

# Check if there were any failures during make or testing
if ( $makeFailCount > 0 || $testDriverFailCount > 0 ) {

    # Display summary of failures
    print("\nMake Failures = $makeFailCount out of $totalMakes");
    print(
"\nTest Driver Failures = $testDriverFailCount out of $totalTestDrivers\n"
    );
    print( "\nMakes that failed:\n  " . join( "\n  ", @makesFailed ) . "\n" )
      if $makeFailCount;
    print( "\nTests that failed:\n  " . join( "\n  ", @testsFailed ) . "\n" )
      if $testDriverFailCount;
}
else {

    # Display message of all makes and tests pass to user if silent option
    # not specified
    print(
"\nAll $totalTestDrivers Test Drivers Passed with $totalTestpoints Testpoints.\n"
      )
      if ( !$silent );
}

# Exit with the number of tests that failed
exit($testDriverFailCount);

################################################################################
#
# SUBROUTINE: worm
#
#     Description:  This subroutine will perform the necessary unit tests be
#                   calling makeTestDrivers and executeTestDrivers  subroutines
#                   and then check each subdirectory below the base directory
#                   recursively.
#
#     Parameter(s):  base directory to start testing
#
#     Return:  None
#
################################################################################

sub worm {

    # Assign local variable to input parameter
    local ($base_dir) = @_;
    local ( @files, $i );

    # Invoke subroutine to make test driver in the base directory
    &makeTestDrivers($base_dir);

    # Invoke subroutine to execute tests in the base directory
    &executeTestDrivers($base_dir);

    # Create array of entries found in directory
    @files = <*>;

    # Loop through the file list looking for another directory that is not
    # labelled CVS and recursively invoke subroutine worm
    $i = 0;
    while ( $files[$i] ) {

        # Check for directory and directory not labelled CVS
        if (   -d $files[$i]
            && ( $files[$i] ne "CVS" )
            && ( $files[$i] ne "temp" )
            && ( $files[$i] ne "verified" ) )
        {

            # Change current directory to directory found
            chdir( $files[$i] );

            # Invoke subroutine worm again
            &worm( $files[$i] );

            # Change current directory back to parent
            chdir("..");
        }

        # Increment file list index
        $i++;
    }
}

################################################################################
#
#  SUBROUTINE: makeTestDrivers
#
#      Description:  This subroutine will perform a make for all the necessary
#                    test drivers.    This will occur in the specified
#                    base directory which is passed as the only parameter.
#
#      Parameter(s):  base directory where make and test drivers are located
#
#      Return:  None
#
################################################################################

sub makeTestDrivers {
    local ($base_dir) = @_;
    local ($pwd);

    # Set variable pwd to current working directory
    $pwd = cwd;

    # Display message to user in which directory testing is taken place only
    # if the silent command option was not specified
    print("---- Entering $pwd ----\n") if ( !$silent );

    # Check for the existence of a file name Makefile in current directory
    if ( -e "Makefile" ) {

        # Increment the total number of makes executed
        $totalMakes++;

        if ($clean) {

            # Execute the make clean
            `make clean`;

            # Execute the make distclean
            `make distclean`;
        }

        # Execute the make and save results
        $_ = join( "\n|| ", split( "\n", "\n" . `make` ) );

        # Check the output of make for return value != 0 or any of the
        # following words: FAILED, FAULT, ERROR, Not found, SIGNAL
        if (   ( $? != 0 )
            || m/FAILED/i
            || m/FAULT/i
            || m/\bERROR/i
            || m/Not found/i
            || m/SIGNAL/i )
        {

            # Display the errored output of make if silent option not enabled
            print("$_\n") if ( !$silent );

            # Display the make failed in the current directory
            print("\nMake for $pwd Failed\n");

            # Increment the number of makes that have failed
            $makeFailCount++;

            # Push the current working directory onto the list of failed
            # make directories list
            push( @makesFailed, $pwd );
        }
        else {

            # Display the results of the successful make if verbose option set
            print("$_\n") if $verbose;

            # Display the make was successful if silent option not set
            print("\nMake successful.\n") if ( !$silent );
            if ($clean) {
                `make install`;
            }
        }
    }
}

################################################################################
#
#  SUBROUTINE: executeTestDrivers
#
#      Description:  This subroutine will execute all the necessary
#                    test drivers.    This will occur in the specified
#                    base directory which is passed as the only parameter.
#
#      Parameter(s):  base directory where make and test drivers are located
#
################################################################################

sub executeTestDrivers {
    local ($base_dir) = @_;
    local ( @files, $j );
    local ($pwd);
    local ($exitValue) = 0;

    # Set variable to pwd to current test directory
    $pwd = cwd;

    # Create a list of all elements in base directory
    @files = <*>;

    # Loop through all the items in the files array
    $initialTest = 0;
    $j           = 0;
    while ( $files[$j] ) {

        # Check that the item is not a directory and is executable and
        # conforms to the test driver naming convention TST
        if (   !( -d $files[$j] )
            && ( -x $files[$j] )
            && ( $files[$j] =~ /^TST/i || $files[$j] =~ /^ATST/i ) )
        {

            # Increment total number of test drivers executed
            $totalTestDrivers++;

            # Perform subdirectory checks only the first time a test
            # file is found with the list of enteries
            if ( $initialTest == 0 ) {
                $initialTest++;

                # Check for a temp subdirectory already exists and if not
                # then create one
                &checkForTempDirectory();

                # Check for the presence of verified directory
                if ( &checkForVerifiedDirectory() ) {

    # Display message to user that verified directories do not
    # exist eventhough TST files have been found
    #                    print("Unable to execute test drivers in $base_dir\n");
                    print("No verified subdirectory present.\n") if ($verbose);

               # Increment test fail count
               #                   $testDriverFailCount++;
               # Save the file failed
               #                    push(@testsFailed, $pwd . "/" . $files[$j]);
               #                    return;
                }
            }

            # Display message to user of which test driver is being executed
            print("--- Executing test driver $files[$j]\n") if ( !$silent );

            # Execute the test driver
            system(
"./$files[$j] 1> temp/$files[$j].stdout 2> temp/$files[$j].stderr"
            );
            $retVal = $?;

            # Count testpoints
            $testPattern = "\"\\*\\*\\*\\* TESTPOINT \\*\\*\\*\\*\"";
            $totalPoints = `grep -c $testPattern temp/$files[$j].stdout`;
            $totalPoints += `grep -c $testPattern temp/$files[$j].stderr`;
            $failPoints =
              `grep "> TESTPOINT FAILED" temp/$files[$j].stdout | wc -l`;
            $failPoints +=
              `grep "> TESTPOINT FAILED" temp/$files[$j].stderr | wc -l`;
            $testpointFailCount += $failPoints;
            $totalTestpoints    += $totalPoints;

            # Check result of test driver
            if (   ( ( $retVal != 0 ) && ( $files[$j] =~ /^TST/i ) )
                || ( ( $retVal == 0 ) && ( $files[$j] =~ /^ATST/i ) ) )
            {

                # Display test driver failed
                $failPoints++;
                print(
                    "Test driver: $files[$j] Failed (Return value $retVal).\n");

                # Increment the total number of test failed
                $testDriverFailCount++;

                # Push the current working directory on the list of failed
                push( @testsFailed, $pwd . "/" . $files[$j] );
            }
            else {

               # Create filter versions of STDOUT and STDERR to replace variable
               # items such as date, time and host
                &filterStdFiles( $files[$j] );

                # Perform difference on STDOUT file collected
                # with verified files
                $exitValue = &compareStream("verified/$files[$j].stdout");
                if ( $exitValue & 2 ) {

                    # STDOUT verified doesn't exist. Search STDOUT capture
                    # for strings indicating error or failure
                    $exitValue |= &errorStrSearch("$files[$j].stdout");
                }
                if ( ( $exitValue & 8 ) || ( $exitValue & 64 ) ) {

                    # Increment the total number of test failed
                    $testDriverFailCount++;

                    # Push test on failed list
                    print(
"Test failed ($failPoints out of $totalPoints testpoints failed)\n"
                      )
                      if ( !$silent );
                    push( @testsFailed, $pwd . "/" . $files[$j] );
                }
                else {

                    # Perform difference on STDERR file collection with verified
                    $exitValue = &compareStream("verified/$files[$j].stderr");
                    if ( $exitValue & 4 ) {

                        # STDERR verified doesn't exist. Search STDERR capture
                        # for strings indicating error or failure
                        $exitValue |= &errorStrSearch("$files[$j].stderr");
                    }
                    if ( ( $exitValue & 16 ) || ( $exitValue & 128 ) ) {

                        # Increment the total number of tests failed
                        $testDriverFailCount++;

                        # Push test on failed list
                        print(
"Test failed ($failPoints out of $totalPoints testpoints failed)\n"
                          )
                          if ( !$silent );
                        push( @testsFailed, $pwd . "/" . $files[$j] );
                    }
                    else {
                        print("Test successful ($totalPoints testpoints)\n")
                          if ( !$silent );
                    }
                }
            }
        }
        $j++;
    }
}

################################################################################
#
#  SUBROUTINE: errorStrSearch
#
#      Description:  This subroutine will search the file specified in its
#                    parameter for error strings and if they do exists
#                    the appropriate value will be returned.
#
#      Parameter(s): fileName - input file to search for strings
#
#      Return:   0  -  No error strings found
#               64  -  Error strings found in STDOUT
#              128  -  Error strings found in STDERR
#
###############################################################################

sub errorStrSearch {
    local ($fileName)  = @_;
    local ($returnVal) = 0;

    # Open the captured file
    open( CAPT_FILE, "<temp/$fileName" );

    # Scan through all the lines of the file searching for error strings
    while (<CAPT_FILE>) {
        if (   m/FAIL/i
            || m/FAULT/i
            || m/ERROR/i
            || m/Not Found/i
            || m/SIGNAL/i
            || m/NO SUCH FILE/i
            || m/\|E\|/i
            || m/\|A\|/i )
        {
            print;
            if ( $fileName =~ m/out/ ) {
                print(
                    "        Failed - File $fileName contains error strings.\n"
                );
                $returnVal = 64;
            }
            elsif ( $fileName =~ m/err/ ) {
                print(
                    "        Failed - File $fileName contains error strings.\n"
                );
                $returnVal = 128;
            }
            last;
        }
    }

    # Close the capture file
    close(CAPT_FILE);

    # Return the return value
    return ($returnVal);
}

################################################################################
#
#  SUBROUTINE: compareStream
#
#      Description:  This subroutine will compare the captured stream file with
#                    a file in the verified directory if one exists.
#
#      Parameter(s): streamFile - input file to compare
#
#      Return:   0  -  Compare successful
#                2  -  STDOUT verified file doesn't exist
#                4  -  STDERR verified file doesn't exist
#                8  -  STDOUT verified file doesn't compare
#               16  -  STDERR verified file doesn't compare
#
###############################################################################

sub compareStream {
    local ($streamFile) = @_;
    local ($returnVal)  = 0;
    local ($tempFile)   = "";

    # Check for existence of verified STD stream files
    if ( !( -e $streamFile ) ) {

        # Display message to user that verified STDOUT file doesn't exist
        print("        File $streamFile doesn't exist.\n") if ($verbose);

        # Set exit value bit 1 to indicate proper failure
        if ( $streamFile =~ /out$/ ) {
            $returnVal |= 2;
        }
        elsif ( $streamFile =~ /err$/ ) {
            $returnVal |= 4;
        }
    }
    else {

        # Verified STD stream file exists

        # Create name of the temp file to compare
        $tempFile = $streamFile;
        $tempFile =~ s/verified/temp/;
        $tempFile = $tempFile . ".mod";

        # Perform difference on the STD stream files
        $diffstdout = `diff $streamFile $tempFile`;

        # Check the return value of the difference
        if ( $? != 0 ) {

            # Difference of STD stream files failed

            # Check for STDOUT in file name to convey appropirate return value
            if ( $streamFile =~ /out$/ ) {

                # Display message of the failure of difference to user
                print("        Failed - STDOUT difference\n$diffstdout\n");

                # Exit value to indicate STDOUT did not compare
                $returnVal |= 8;
            }
            elsif ( $streamFile =~ /err$/ ) {

                # Display message of the failure of difference to user
                print("        Failed - STDERR difference\n$diffstdout\n");

                # Exit value to indicate STDERR did not compare
                $returnVal |= 16;
            }
        }
    }

    # Return the result of the compare
    return ($returnVal);
}

################################################################################
#
#  SUBROUTINE: checkForTempDirectory
#
#      Description:  This subroutine will check for the existence of a temp
#                    directory to store STDOUT, STDERR files during test.  If
#                    the subdirectory doesn't exist, it will be created.
#
#      Parameter(s):  None
#
#      Return: None
#
################################################################################

sub checkForTempDirectory {

    # Check if a temp directory already exists in the current work directory
    if ( !( -e "temp" ) ) {

        # Create temp directory to store STDOUT, STDERR files during test
        `mkdir temp`;

        # Display message of new directory created
        print("Creating temp directory\n") if ($verbose);
    }
}

################################################################################
#
#  SUBROUTINE: checkForVerifiedDirectory
#
#      Description:  This subroutine will check for the existence of a verified
#                    directory which stores verified STDOUT, STDERR files for
#                    comparison during the test.
#
#      Parameter(s):  None
#
#      Return: 0 - subdirectory present
#              1 - subdirectory not present
#
################################################################################

sub checkForVerifiedDirectory {
    $returnValue = 0;

    # Check if verified subdirectory exists in the current working directory
    if ( !( -e "verified" ) ) {

        # Set return value to 1
        $returnValue = 1;
    }
    return ($returnValue);
}

################################################################################
#
#  SUBROUTINE: filterStdFiles
#
#      Description:  This subroutine will filter variable items in the STDOUT
#                    and STDERR files captured.  Date, time and host names
#                    are variable items in the log messages captured.
#
#      Parameter(s):  test file name
#
#      Return: None
#
################################################################################

sub filterStdFiles {

    local ($fileName) = @_;

    # Open the STDOUT file for reading
    open( OUTFILE, "< temp/$fileName.stdout" );

    # Open mod file to place filtered STDOUT
    open( MODFILE, "> temp/$fileName.stdout.mod" );

    # Replace the variable data, time and host information with constants
    $hostname = `hostname`;
    chop $hostname;
    while (<OUTFILE>) {
        s/\s+\d+:\d+:\d+\w/<TIME>/g;
        s/\d+:\d+:\d+/<DATE>/g;
        s/$hostname\s*/<HOST>/g;
        s/: Line \d+/: Line <LINENO>/g;
        s/\:\d+/\:<LINENO>/g;
        s/allocate \d+ bytes at/allocate <N> bytes at/g;

        # Filter lines with *** malloc.  This is an artifact of Mac testing of
        # memory functions
        if ( !m/\*\*\*\smalloc/ ) {
            print MODFILE ($_);
        }
    }

    # Close mod file
    close(MODFILE);

    # Close STDERR file
    close(OUTFILE);

    # Open the STDERR file for reading
    open( OUTFILE, "< temp/$fileName.stderr" );

    # Open mod file to place filtered STDERR
    open( MODFILE, "> temp/$fileName.stderr.mod" );

    # Replace the variable date, time and host information with constants
    while (<OUTFILE>) {
        s/\s+\d+:\d+:\d+\w/<TIME>/g;
        s/\d+:\d+:\d+/<DATE>/g;
        s/$hostname\s*/<HOST>/g;
        s/: Line \d+/: Line <LINENO>/g;
        s/\:\d+/\:<LINENO>/g;
        s/allocate \d+ bytes at/allocate <N> bytes at/g;

        # Filter lines with *** malloc.  This is an artifact of Mac testing of
        # memory functions
        if ( !m/\*\*\*\smalloc/ ) {
            print MODFILE ($_);
        }
    }

    # Close mod file
    close(MODFILE);

    # Close STDERR file
    close(OUTFILE);
}

