diff -r 000000000000 -r 5d03bc08d59c graphicsdeviceinterface/directgdi/test/scripts/testcase.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graphicsdeviceinterface/directgdi/test/scripts/testcase.py Tue Feb 02 01:47:50 2010 +0200 @@ -0,0 +1,405 @@ +# Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies). +# All rights reserved. +# This component and the accompanying materials are made available +# under the terms of "Eclipse Public License v1.0" +# which accompanies this distribution, and is available +# at the URL "http://www.eclipse.org/legal/epl-v10.html". +# +# Initial Contributors: +# Nokia Corporation - initial contribution. +# +# Contributors: +# +# Description: +# + +""" +Test Case + +Container for test cases and interfaces for executing the cases. + +""" +import os +import os.path +import fileinput +import string + +from refimage import RefImage +from htmlreport import * + +# The threshold value for the failure monitor report +FAILURE_THRESHOLD_MONITOR = 5 + +# The expected type of images for testing +KImageType = ".bmp" +KImageTypeAlpha = "-alpha.bmp" + +# Relative path for reference images +KRefPath = "ref\\" + +# Relative path for test images +KTestPath = "test\\" + +# A mid name for diff images will be added to make the diff images names +KDiffImageMidName = "diffImg" + +# HTML file name for an index page of the test results +KIndexPageName = "testresults.html" +# HTML file name for failed images +KPageForFailedImages = "failed.html" +# HTML file name for passed images +KPageForPassedImages = "passed.html" +KPageForFailedAtImages = "failedAt.html" +# HTML file name for a summary page where passed and failed image information is summarised +KPageForSummary = "summary.html" + +# HTML file name for a summary page of the test results for the overnight build results page +KONBSummary = "directgditest_image_results.htm" + +class TestCase: + # Enum of test results + RESULT_PASSED = 1 + RESULT_FAILED = 2 + + def __init__(self, aName, aResultDir, aLogDir, aOnbDir): + self.name = aName + self.resultTestImages = [] + self.resultRefImages = [] + self.resultTestImagesAlpha = [] + self.resultRefImagesAlpha = [] + self.resultTestImagesWithAlphaPair = [] + self.resultRefImagesWithAlphaPair = [] + self.resultDir = aResultDir + self.logDir = aLogDir + self.onbDir = aOnbDir + self.testPassedNumber = 0 + self.testFailedNumber = 0 + self.testFailedAtNumber = 0 + self.passedTestImg = [] + self.failedTestImg = [] + self.failedAtTestImg = [] + self.passedRefImg = [] + self.failedRefImg = [] + self.failedAtRefImg = [] + self.thumbnails = {} + + # Read test images to self.resultRefImages + def readRefImage(self, file): + # @param file A log file to store error messages + refimageDir = os.path.join(self.resultDir, KRefPath) + self.resultRefImages = [] + self.resultRefImagesAlpha = [] + images = os.listdir(refimageDir) + + + #find bitmaps describing alpha channel first + for imageFile in images: + if lower(imageFile[-10:]) == KImageTypeAlpha: + self.resultRefImagesAlpha.append(imageFile) + + #find all bitmaps in directory but those with no alpha pair + for imageFile in images: + alphapair = False + # only process the image with the defined imageType + if lower(imageFile[-4:]) == KImageType and lower(imageFile[-10:]) != KImageTypeAlpha: + alpharefimg = self.formatNameAddAlpha(imageFile) + for imgalph in self.resultRefImagesAlpha: + if alpharefimg == imgalph: + alphapair = True + break + + if alphapair == False: + self.resultRefImages.append(imageFile) + else: + self.resultRefImagesWithAlphaPair.append(imageFile) + + + if self.resultRefImages == []: + print("WARNING: No ref images found at " + self.resultDir + KRefPath) + file.write("warning: No ref images found at: " + self.resultDir + KRefPath + "
") + + # Read test images to self.resultTestImages + # @param file A log file to store error messages + def readTestImage(self, file): + testimageDir = os.path.join(self.resultDir, KTestPath) + self.resultTestImages = [] + self.resultTestImagesAlpha = [] + images = os.listdir(testimageDir) + + #find bitmaps describing alpha channel first + for imageFile in images: + if lower(imageFile[-10:]) == KImageTypeAlpha: + self.resultTestImagesAlpha.append(imageFile) + + #find all bitmaps in directory but those with no alpha pair + for imageFile in images: + alphapair = False + # only process the image with the defined imageType + if lower(imageFile[-4:]) == KImageType and lower(imageFile[-10:]) != KImageTypeAlpha: + alphatestimg = self.formatNameAddAlpha(imageFile) + for imgalph in self.resultTestImagesAlpha: + if alphatestimg == imgalph: + alphapair = True + break + + if alphapair == False: + self.resultTestImages.append(imageFile) + else: + self.resultTestImagesWithAlphaPair.append(imageFile) + + + if self.resultTestImages == []: + print("WARNING: No test images found at " + self.resultDir + KTestPath) + file.write("warning: No test images found at: " + self.resultDir + KTestPath + "
") + + # Remove the prefix of "Bitgdi_" or "Directgdi_" in the image names + # @param aFileName The image file name + def formatBaseName(self, aFileName): + completeName = os.path.basename(aFileName) + index = completeName.find("_") + return completeName[index + 1:] + + # construct name with -alpha postfix + # @param aFileName The image file name + def formatNameAddAlpha(self,aFileName): + (alphaimgname,alphaimgext) = os.path.splitext(aFileName) + return alphaimgname + "-alpha" + alphaimgext + + # construct name by removing -alpha postfix + # @param aFileName The image file name + def formatNameRemoveAlpha(self,aFileName): + (alphaimgname,alphaimgext) = os.path.splitext(aFileName) + index = alphaimgname.find("-alpha") + if index == -1: + return aFileName + else: + return alphaimgname[:index] + alphaimgext + + def processLine(self, aLine): + return string.split(aLine) + + # Compute the difference of test images against their reference images one by one + # @param file A log file to store error messages + def computeDifferences(self, file): + + # Generate list of threshold mappings - + # Consisting of base name & threshold values + thresholds = [] + lineTuple = [] + try: + for line in fileinput.input("thresholds.txt"): + lineTuple = self.processLine(line) + if lineTuple: + try: + if (lineTuple[0] and (lineTuple[0][0] != '#')): + thresholds.append(lineTuple[0]) + thresholds.append(lineTuple[1]) + print "Found threshold exception: ", lineTuple[0], " ", lineTuple[1] + except Exception: + pass + except Exception: + pass + + # Create file to recieve information on any test failures. + # Format: + # \n + failuresFile = open ( 'testFailures.txt', 'w' ) + + # loop through test images + for imgFile in self.resultTestImages: + + # get the test image file basename + #baseName_test = os.path.basename(imgFile) + baseName_test = self.formatBaseName(imgFile) + + # only process the image with the defined imageType + if lower(baseName_test[-4:]) == KImageType: + + # look through reference images + for refFile in self.resultRefImages: + + # get the reference image file basename + #baseName_ref = os.path.basename(refFile) + baseName_ref = self.formatBaseName(refFile) + + # only process the image with the defined imageType + if lower(baseName_ref[-4:]) == KImageType: + + # match the file names of a test image with its reference image + if baseName_test == baseName_ref: + + # create a RefImage and compute difference between test and reference images + test = RefImage(refFile, imgFile, self.resultDir, KDiffImageMidName) + test.computeDifferences(file) + test.makeDiffImages(self.resultDir) + + thresholdValue = -1 + baseCaseName = baseName_test[:-4] + try: + thresholdValue = thresholds[thresholds.index(baseCaseName) + 1] + except Exception: + pass + + test.printResult(file, int(thresholdValue)) + result = test.getResult(file, int(thresholdValue)) + + pyramidValue = test.getPyramidResultValue (); + if result == True: + self.testPassedNumber = self.testPassedNumber + 1 + self.passedTestImg.append((imgFile, pyramidValue)) + self.passedRefImg.append((refFile, pyramidValue)) + + # If the test passes but is above and independant threshold + # record the test and reference image names for report generation. + if (pyramidValue > FAILURE_THRESHOLD_MONITOR): + self.failedAtTestImg.append((imgFile, pyramidValue)) + self.failedAtRefImg.append((refFile, pyramidValue)) + self.testFailedAtNumber = self.testFailedAtNumber + 1 + + else: + # Output failure information to failures file. + strList = [] + strList.append(baseCaseName) + strList.append(`test.pyramidValue()`) + strList.append('\n') + failuresFile.write(' '.join(strList)) + print "Test image failure: ", ' '.join(strList) + self.testFailedNumber = self.testFailedNumber + 1 + self.failedTestImg.append((imgFile, pyramidValue)) + self.failedRefImg.append((refFile, pyramidValue)) + + # loop through test images with alpha pair + for imgFile in self.resultTestImagesWithAlphaPair: + + # get the test image file basename + #baseName_test = os.path.basename(imgFile) + baseName_test = self.formatBaseName(imgFile) + + # only process the image with the defined imageType + if lower(baseName_test[-4:]) == KImageType: + + # look through reference images + for refFile in self.resultRefImagesWithAlphaPair: + + # get the reference image file basename + #baseName_ref = os.path.basename(refFile) + baseName_ref = self.formatBaseName(refFile) + + # only process the image with the defined imageType + if lower(baseName_ref[-4:]) == KImageType: + + # match the file names of a test image with its reference image + if baseName_test == baseName_ref: + + # create a RefImage and compute difference between test and reference images + test = RefImage(refFile, imgFile, self.resultDir, KDiffImageMidName) + test.computeDifferences(file) + test.makeDiffImages(self.resultDir) + + thresholdValue = -1 + baseCaseName = baseName_test[:-4] + try: + thresholdValue = thresholds[thresholds.index(baseCaseName) + 1] + except Exception: + pass + + test.printResult(file, int(thresholdValue)) + result = test.getResult(file, int(thresholdValue)) + + alpharefimg = self.formatNameAddAlpha(refFile) + for imgrefalpha in self.resultRefImagesAlpha: + if alpharefimg == imgrefalpha: + for imgtestalpha in self.resultTestImagesAlpha: + if self.formatBaseName(imgrefalpha) == self.formatBaseName(imgtestalpha): + testalpha = RefImage(imgrefalpha, imgtestalpha, self.resultDir, KDiffImageMidName) + testalpha.computeDifferences(file) + testalpha.makeDiffImages(self.resultDir) + testalpha.printResult(file, int(thresholdValue)) + resultalpha = testalpha.getResult(file, int(thresholdValue)) + if resultalpha == False: + result = False + break + + pyramidValue = test.getPyramidResultValue (); + if result == True: + self.testPassedNumber = self.testPassedNumber + 1 + self.passedTestImg.append((imgFile, pyramidValue)) + self.passedRefImg.append((refFile, pyramidValue)) + + # If the test passes but is above and independant threshold + # record the test and reference image names for report generation. + if (pyramidValue > FAILURE_THRESHOLD_MONITOR): + self.failedAtTestImg.append((imgFile, pyramidValue)) + self.failedAtRefImg.append((refFile, pyramidValue)) + self.testFailedAtNumber = self.testFailedAtNumber + 1 + + else: + # Output failure information to failures file. + strList = [] + strList.append(baseCaseName) + strList.append(`test.pyramidValue()`) + strList.append('\n') + failuresFile.write(' '.join(strList)) + print "Test image alpha-pair failure: ", ' '.join(strList) + self.testFailedNumber = self.testFailedNumber + 1 + self.failedTestImg.append((imgFile, pyramidValue)) + self.failedRefImg.append((refFile, pyramidValue)) + + failuresFile.close() + print "All cases: ", self.testPassedNumber + self.testFailedNumber + print "Failed cases: ", self.testFailedNumber + print "Passed cases: ", self.testPassedNumber + print 'Failed at threshold %d cases: %d' % (FAILURE_THRESHOLD_MONITOR, self.testFailedAtNumber) + + # Generate a report of the test results + self.generateReport(os.path.join(self.resultDir, KTestPath), os.path.join(self.resultDir, KRefPath), self.resultDir) + + # Generate a report of test results + # @param aTestDir The directory of the test images + # @param aRefDir The directory of the reference images + # @param diffDir The directory of the diff images + def generateReport(self, aTestDir, aRefDir, aDiffDir): + # Write a main index page of the test results + report = file(self.logDir + KIndexPageName, "wt") + writeHtmlHeader(report, self.testFailedNumber, self.testPassedNumber, self.testFailedAtNumber, FAILURE_THRESHOLD_MONITOR) + writeHtmlFooter(report) + report.close() + + #sort tables in alphabetical order + self.failedTestImg.sort() + self.failedRefImg.sort() + self.passedTestImg.sort() + self.passedRefImg.sort() + self.failedAtTestImg.sort() + self.failedAtRefImg.sort() + + summary = file(self.logDir + KPageForSummary, "wt") + writeFailedDetails(summary, self.failedTestImg, self.failedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir) + writePassedDetails(summary, self.passedTestImg, self.passedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir) + writeFailedAtDetails(summary, self.failedAtTestImg, self.failedAtRefImg, aTestDir, aRefDir, aDiffDir, self.logDir, FAILURE_THRESHOLD_MONITOR) + + # Write a html page of the failed images + failedDetail = file(self.logDir + KPageForFailedImages, "wt") + writeSimpleHtmlHeader(failedDetail) + writeFailedDetails(failedDetail, self.failedTestImg, self.failedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir) + writeHtmlFooter(failedDetail) + failedDetail.close() + + # Write a html page of the passed images + passedDetail = file(self.logDir + KPageForPassedImages, "wt") + writeSimpleHtmlHeader(passedDetail) + writePassedDetails(passedDetail, self.passedTestImg, self.passedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir) + writeHtmlFooter(passedDetail) + passedDetail.close() + + # Write a html page of the failed at images. + failedAtDetail = file(self.logDir + KPageForFailedAtImages, "wt") + writeSimpleHtmlHeader(failedAtDetail) + writeFailedAtDetails(failedAtDetail, self.failedAtTestImg, self.failedAtRefImg, aTestDir, aRefDir, aDiffDir, self.logDir, FAILURE_THRESHOLD_MONITOR) + writeHtmlFooter(failedAtDetail) + failedAtDetail.close() + + # Write a summary page for the overnight build results + onb_report = file(self.onbDir + KONBSummary, "wt") + writeHtmlONB(onb_report, self.testFailedNumber, self.testPassedNumber, self.testFailedAtNumber, FAILURE_THRESHOLD_MONITOR, KIndexPageName) + onb_report.close()