#!/usr/bin/perl

# Copyright (C) 2005, 2006, 2007 Apple Inc. All rights reserved.

# Copyright (C) 2006 Alexey Proskuryakov (ap@nypop.com)

# Copyright (C) 2007 Matt Lilek (pewtermoose@gmail.com)

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

#

# 1.  Redistributions of source code must retain the above copyright

#     notice, this list of conditions and the following disclaimer. 

# 2.  Redistributions in binary form must reproduce the above copyright

#     notice, this list of conditions and the following disclaimer in the

#     documentation and/or other materials provided with the distribution. 

# 3.  Neither the name of Apple Computer, Inc. ("Apple") nor the names of

#     its contributors may be used to endorse or promote products derived

#     from this software without specific prior written permission. 

#

# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY

# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY

# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# Script to run the Web Kit Open Source Project layout tests.

# Run all the tests passed in on the command line.

# If no tests are passed, find all the .html, .shtml, .xml, .xhtml, .pl, .php (and svg) files in the test directory.

# Run each text.

# Compare against the existing file xxx-expected.txt.

# If there is a mismatch, generate xxx-actual.txt and xxx-diffs.txt.

# At the end, report:

#   the number of tests that got the expected results

#   the number of tests that ran, but did not get the expected results

#   the number of tests that failed to run

#   the number of tests that were run but had no expected results to compare against

use strict;

use warnings;

use Cwd;

use File::Basename;

use File::Copy;

use File::Find;

use File::Path;

use File::Spec;

use File::Spec::Functions;

use FindBin;

use Getopt::Long;

use IPC::Open2;

use IPC::Open3;

use Time::HiRes qw(time);

use lib $FindBin::Bin;

use webkitdirs;

use POSIX;

sub openDumpTool();

sub closeDumpTool();

sub dumpToolDidCrash();

sub closeHTTPD();

sub countAndPrintLeaks($$$);

sub fileNameWithNumber($$);

sub numericcmp($$);

sub openHTTPDIfNeeded();

sub pathcmp($$);

sub processIgnoreTests($);

sub slowestcmp($$);

sub splitpath($);

sub stripExtension($);

sub isTextOnlyTest($);

sub expectedDirectoryForTest($;$);

sub printFailureMessageForTest($$);

sub toURL($);

sub toWindowsPath($);

sub closeCygpaths();

sub validateSkippedArg($$;$);

sub htmlForExpectedAndActualResults($);

sub deleteExpectedAndActualResults($);

sub recordActualResultsAndDiff($$);

sub buildPlatformHierarchy();

# Argument handling

my $shouldCheckLeaks = '';

my $configuration = configuration();

my $guardMalloc = '';

my $httpdPort = 8000;

my $httpdSSLPort = 8443;

my $ignoreTests = '';

my $launchSafari = 1;

my $platform;

my $pixelTests = '';

my $quiet = '';

my $repaintSweepHorizontally = '';

my $repaintTests = '';

my $report10Slowest = 0;

my $resetResults = 0;

my $showHelp = 0;

my $testsPerDumpTool = 1000;

my $testHTTP = 1;

my $testOnlySVGs = '';

my $testResultsDirectory = "/tmp/layout-test-results";

my $threaded = 0;

my $treatSkipped = "default";

my $verbose = 0;

my $useValgrind = 0;

my $strictTesting = 0;

my $generateNewResults = 1;

my $stripEditingCallbacks = isCygwin();

my $root;

my $expectedTag = "expected";

my $actualTag = "actual";

my $diffsTag = "diffs";

my $errorTag = "stderr";

if (isTiger()) {

    $platform = "mac-tiger";

} elsif (isLeopard()) {

    $platform = "mac-leopard";

} elsif (isOSX()) {

    $platform = "mac";

} elsif (isQt()) {

    $platform = "qt";

} elsif (isGtk()) {

    $platform = "gtk";

} elsif (isCygwin()) {

    $platform = "win";

}

if (!defined($platform)) {

    print "WARNING: Your platform is not recognized. Any platform-specific results will be generated in platform/undefined.\n";

    $platform = "undefined";

}

my $programName = basename($0);

my $launchSafariDefault = $launchSafari ? "launch" : "do not launch";

my $httpDefault = $testHTTP ? "run" : "do not run";

my $usage = <<EOF;

Usage: $programName [options] [testdir|testpath ...]

  -c|--configuration config       Set DumpRenderTree build configuration

  -g|--guard-malloc               Enable malloc guard

  --help                          Show this help message

  -h|--horizontal-sweep           Change repaint to sweep horizontally instead of vertically (implies --repaint-tests)

  --[no-]http                     Run (or do not run) http tests (default: $httpDefault)

  -i|--ignore-tests               Comma-separated list of directories or tests to ignore

  --[no-]launch-safari            Launch (or do not launch) Safari to display test results (default: $launchSafariDefault)

  -l|--leaks                      Enable leaks checking

  --[no-]new-test-results         Generate results for new tests

  -p|--pixel-tests                Enable pixel tests

  --platform                      Override the detected platform to use for tests and results (default: $platform)

  --port                          Web server port to use with http tests

  -q|--quiet                      Less verbose output

  -r|--repaint-tests              Run repaint tests (implies --pixel-tests)

  --reset-results                 Reset ALL results (including pixel tests if --pixel-tests is set)

  -o|--results-directory          Output results directory (default: $testResultsDirectory)

  --root                          Path to root tools build

  -1|--singly                     Isolate each test case run (implies --verbose)

  --skipped=[default|ignore|only] Specifies how to treat the Skipped file

                                     default: Tests/directories listed in the Skipped file are not tested

                                     ignore:  The Skipped file is ignored

                                     only:    Only those tests/directories listed in the Skipped file will be run

  --slowest                       Report the 10 slowest tests

  --strict                        Do a comparison with the output on Mac (Qt only)

  --[no-]strip-editing-callbacks  Remove editing callbacks from expected results

  --svg                           Run only SVG tests (implies --pixel-tests)

  -t|--threaded                   Run a concurrent JavaScript thead with each test

  --valgrind                      Run DumpRenderTree inside valgrind (Qt/Linux only)

  -v|--verbose                    More verbose output (overrides --quiet)

EOF

my $getOptionsResult = GetOptions(

    'c|configuration=s' => \$configuration,

    'debug|devel' => sub { $configuration = "Debug" },

    'guard-malloc|g' => \$guardMalloc,

    'help' => \$showHelp,

    'horizontal-sweep|h' => \$repaintSweepHorizontally,

    'http!' => \$testHTTP,

    'ignore-tests|i=s' => \$ignoreTests,

    'launch-safari!' => \$launchSafari,

    'leaks|l' => \$shouldCheckLeaks,

    'pixel-tests|p' => \$pixelTests,

    'platform=s' => \$platform,

    'port=i' => \$httpdPort,

    'quiet|q' => \$quiet,

    'release|deploy' => sub { $configuration = "Release" },

    'repaint-tests|r' => \$repaintTests,

    'reset-results' => \$resetResults,

    'new-test-results!' => \$generateNewResults,

    'results-directory|o=s' => \$testResultsDirectory,

    'singly|1' => sub { $testsPerDumpTool = 1; },

    'nthly=i' => \$testsPerDumpTool,

    'skipped=s' => \&validateSkippedArg,

    'slowest' => \$report10Slowest,

    'svg' => \$testOnlySVGs, 

    'threaded|t' => \$threaded,

    'verbose|v' => \$verbose,

    'valgrind' => \$useValgrind,

    'strict' => \$strictTesting,

    'strip-editing-callbacks!' => \$stripEditingCallbacks,

    'root=s' => \$root,

);

if (!$getOptionsResult || $showHelp) {

    print STDERR $usage;

    exit 1;

}

my $ignoreSkipped = $treatSkipped eq "ignore";

my $skippedOnly = $treatSkipped eq "only";

!$skippedOnly || @ARGV == 0 or die "--skipped=only cannot be used when tests are specified on the command line.";

setConfiguration($configuration);

my $configurationOption = "--" . lc($configuration);

$repaintTests = 1 if $repaintSweepHorizontally;

$pixelTests = 1 if $testOnlySVGs;

$pixelTests = 1 if $repaintTests;

$verbose = 1 if $testsPerDumpTool == 1;

if ($shouldCheckLeaks && $testsPerDumpTool > 1000) {

    print STDERR "\nWARNING: Running more than 1000 tests at a time with MallocStackLogging enabled may cause a crash.\n\n";

}

# Force --no-http for Qt/Linux, for now.

$testHTTP = 0 if isQt();

setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));

my $productDir = productDir();

$productDir .= "/bin" if (isQt());

chdirWebKit();

if(!defined($root)){

    my $buildResult;

    if (isCygwin()) {

        $buildResult = buildVisualStudioProject("WebKitTools/DumpRenderTree/DumpRenderTree.sln");

    } else {

        # Push the parameters to build-dumprendertree as an array

        my @args;

        push(@args, $configuration);

        if (isQt()) {

            push(@args, "--qt");

        } elsif (isGtk()) {

            push(@args, "--gtk");

        }

        $buildResult = system "WebKitTools/Scripts/build-dumprendertree", @args;

    }

    if ($buildResult) {

        print STDERR "Compiling DumpRenderTree failed!\n";

        exit WEXITSTATUS($buildResult);

    }

}

my $dumpToolName = "DumpRenderTree";

$dumpToolName .= "_debug" if isCygwin() && $configuration ne "Release";

my $dumpTool = "$productDir/$dumpToolName";

die "can't find executable $dumpToolName (looked in $productDir)\n" unless -x $dumpTool;

my $imageDiffTool = "$productDir/ImageDiff";

die "can't find executable $imageDiffTool (looked in $productDir)\n" if $pixelTests && !-x $imageDiffTool;

checkFrameworks() unless isCygwin();

my $layoutTestsName = "LayoutTests";

my $testDirectory = File::Spec->rel2abs($layoutTestsName);

my $expectedDirectory = $testDirectory;

my $platformBaseDirectory = catdir($testDirectory, "platform");

my $platformTestDirectory = catdir($platformBaseDirectory, $platform);

my @platformHierarchy = buildPlatformHierarchy();

$expectedDirectory = $ENV{"WebKitExpectedTestResultsDirectory"} if $ENV{"WebKitExpectedTestResultsDirectory"};

if ($testOnlySVGs) {

    $testDirectory .= "/svg";

    $expectedDirectory .= "/svg";

}

my $testResults = catfile($testResultsDirectory, "results.html");

print "Running tests from $testDirectory\n";

my @tests = ();

my %testType = ();

system "ln", "-s", $testDirectory, "/tmp/LayoutTests" unless -x "/tmp/LayoutTests";

my %ignoredFiles = ();

my %ignoredDirectories = map { $_ => 1 } qw(platform);

my %ignoredLocalDirectories = map { $_ => 1 } qw(.svn _svn resources);

my %supportedFileExtensions = map { $_ => 1 } qw(html shtml xml xhtml pl php);

if ($testOnlySVGs) {

    %supportedFileExtensions = map { $_ => 1 } qw(svg xml);

} elsif (checkWebCoreSVGSupport($testOnlySVGs)) { 

    $supportedFileExtensions{'svg'} = 1;

} elsif (isCygwin()) {

    # FIXME: We should fix webkitdirs.pm:hasSVGSupport() to do the correct

    # check for Windows instead of forcing this here.

    $supportedFileExtensions{'svg'} = 1;

} else {

    $ignoredLocalDirectories{'svg'} = 1;

}

if (!$testHTTP) {

    $ignoredDirectories{'http'} = 1;

}

if ($ignoreTests) {

    processIgnoreTests($ignoreTests);

}

if (!$ignoreSkipped) {

    foreach my $level (@platformHierarchy) {

        if (open SKIPPED, "<", "$level/Skipped") {

            if ($verbose && !$skippedOnly) {

                my ($dir, $name) = splitpath($level);

                print "Skipped tests in $name:\n";

            }

            while (<SKIPPED>) {

                my $skipped = $_;

                chomp $skipped;

                $skipped =~ s/^[ \n\r]+//;

                $skipped =~ s/[ \n\r]+$//;

                if ($skipped && $skipped !~ /^#/) {

                    if ($skippedOnly) {

                        push(@ARGV, $skipped);

                    } else {

                        if ($verbose) {

                            print "    $skipped\n";

                        }

                        processIgnoreTests($skipped);

                    }

                }

            }

            close SKIPPED;

        }

    }

}

my $directoryFilter = sub {

    return () if exists $ignoredLocalDirectories{basename($File::Find::dir)};

    return () if exists $ignoredDirectories{File::Spec->abs2rel($File::Find::dir, $testDirectory)};

    return @_;

};

my $fileFilter = sub {

    my $filename = $_;

    if ($filename =~ /\.([^.]+)$/) {

        if (exists $supportedFileExtensions{$1}) {

            my $path = File::Spec->abs2rel(catfile($File::Find::dir, $filename), $testDirectory);

            push @tests, $path if !exists $ignoredFiles{$path};

        }

    }

};

for my $test (@ARGV) {

    $test =~ s/^($layoutTestsName|$testDirectory)\///;

    my $fullPath = catfile($testDirectory, $test);

    if (file_name_is_absolute($test)) {

        print "can't run test $test outside $testDirectory\n";

    } elsif (-f $fullPath) {

        my ($filename, $pathname, $fileExtension) = fileparse($test, qr{\.[^.]+$});

        if (!exists $supportedFileExtensions{substr($fileExtension, 1)}) {

            print "test $test does not have a supported extension\n";

        } elsif ($testHTTP || $pathname !~ /^http\//) {

            push @tests, $test;

        }

    } elsif (-d $fullPath) {

        find({ preprocess => $directoryFilter, wanted => $fileFilter }, $fullPath);

        for my $level (@platformHierarchy) {

            my $platformPath = catfile($level, $test);

            find({ preprocess => $directoryFilter, wanted => $fileFilter }, $platformPath) if (-d $platformPath);

        }

    } else {

        print "test $test not found\n";

    }

}

if (!scalar @ARGV) {

    find({ preprocess => $directoryFilter, wanted => $fileFilter }, $testDirectory);

    for my $level (@platformHierarchy) {

        find({ preprocess => $directoryFilter, wanted => $fileFilter }, $level);

    }

}

die "no tests to run\n" if !@tests;

@tests = sort pathcmp @tests;

my %counts;

my %tests;

my %imagesPresent;

my %durations;

my $count = 0;

my $leaksOutputFileNumber = 1;

my $totalLeaks = 0;

my @toolArgs = ();

push @toolArgs, "--dump-all-pixels" if $pixelTests && $resetResults;

push @toolArgs, "--pixel-tests" if $pixelTests;

push @toolArgs, "--repaint" if $repaintTests;

push @toolArgs, "--horizontal-sweep" if $repaintSweepHorizontally;

push @toolArgs, "--threaded" if $threaded;

push @toolArgs, "--paint" if $shouldCheckLeaks; # Otherwise, DRT won't exercise painting leaks.

push @toolArgs, "-";

$| = 1;

my $imageDiffToolPID;

if ($pixelTests) {

    local %ENV;

    $ENV{MallocStackLogging} = 1 if $shouldCheckLeaks;

    $imageDiffToolPID = open2(\*DIFFIN, \*DIFFOUT, $imageDiffTool, "") or die "unable to open $imageDiffTool\n";

}

my $dumpToolPID;

my $isDumpToolOpen = 0;

my $dumpToolCrashed = 0;

my $atLineStart = 1;

my $lastDirectory = "";

my $isHttpdOpen = 0;

sub catch_pipe { $dumpToolCrashed = 1; }

$SIG{"PIPE"} = "catch_pipe";

print "Testing ", scalar @tests, " test cases.\n";

my $overallStartTime = time;

for my $test (@tests) {

    next if $test eq 'results.html';

    openDumpTool();

    my $base = stripExtension($test);

    if ($verbose) {

        print "running $test -> ";

        $atLineStart = 0;

    } elsif (!$quiet) {

        my $dir = $base;

        $dir =~ s|/[^/]+$||;

        if ($dir ne $lastDirectory) {

            print "\n" unless $atLineStart;

            print "$dir ";

            $lastDirectory = $dir;

        }

        print ".";

        $atLineStart = 0;

    }

    my $result;

    my $startTime = time if $report10Slowest;

    if ($test !~ /^http\//) {

        my $testPath = "$testDirectory/$test";

        if (isCygwin()) {

            $testPath = toWindowsPath($testPath);

        } else {

            $testPath = canonpath($testPath);

        }

        print OUT "$testPath\n";

    } else {

        openHTTPDIfNeeded();

        if ($test !~ /^http\/tests\/local\// && $test !~ /^http\/tests\/ssl\//) {

            my $path = canonpath($test);

            $path =~ s/^http\/tests\///;

            print OUT "http://127.0.0.1:$httpdPort/$path\n";

        } elsif ($test =~ /^http\/tests\/ssl\//) {

            my $path = canonpath($test);

            $path =~ s/^http\/tests\///;

            print OUT "https://127.0.0.1:$httpdSSLPort/$path\n";

        } else {

            my $testPath = "$testDirectory/$test";

            if (isCygwin()) {

                $testPath = toWindowsPath($testPath);

            }

            else {

                $testPath = canonpath($testPath);

            }

            print OUT "$testPath\n";

        }

    }

    my $actual = "";

    while (<IN>) {

        last if /#EOF/;

        $actual .= $_;

    }

    $actual =~ s/\r//g if isCygwin();

    my $isText = isTextOnlyTest($actual);

    $durations{$test} = time - $startTime if $report10Slowest;

    my $expected;

    my $expectedDir = expectedDirectoryForTest($base, $isText);

    if (!$resetResults && open EXPECTED, "<", "$expectedDir/$base-$expectedTag.txt") {

        $expected = "";

        while (<EXPECTED>) {

            next if $stripEditingCallbacks && $_ =~ /^EDITING DELEGATE:/;

            $expected .= $_;

        }

        close EXPECTED;

    }

    my $expectedMac;

    if (!isOSX() && $strictTesting && !$isText) {

      if (!$resetResults && open EXPECTED, "<", "$testDirectory/platform/mac/$base-$expectedTag.txt") {