1 # Copyright (c) 2007-2009 Nokia Corporation and/or its subsidiary(-ies).

     2 # All rights reserved.

     3 # This component and the accompanying materials are made available

     4 # under the terms of "Eclipse Public License v1.0"

     5 # which accompanies this distribution, and is available

     6 # at the URL "http://www.eclipse.org/legal/epl-v10.html".

     7 #

     8 # Initial Contributors:

     9 # Nokia Corporation - initial contribution.

    10 #

    11 # Contributors:

    12 #

    13 # Description:

    14 #

    15 

    16 """

    17 Test Case

    18 

    19 Container for test cases and interfaces for executing the cases.

    20 

    21 """

    22 import os

    23 import os.path

    24 import fileinput

    25 import string

    26 

    27 from refimage import RefImage

    28 from htmlreport import *

    29 

    30 # The threshold value for the failure monitor report

    31 FAILURE_THRESHOLD_MONITOR = 5

    32 

    33 # The expected type of images for testing

    34 KImageType = ".bmp"

    35 KImageTypeAlpha = "-alpha.bmp"

    36 

    37 # Relative path for reference images

    38 KRefPath = "ref\\"

    39 

    40 # Relative path for test images

    41 KTestPath = "test\\"

    42 

    43 # A mid name for diff images will be added to make the diff images names

    44 KDiffImageMidName = "diffImg"

    45 

    46 # HTML file name for an index page of the test results

    47 KIndexPageName = "testresults.html"

    48 # HTML file name for failed images

    49 KPageForFailedImages = "failed.html"

    50 # HTML file name for passed images

    51 KPageForPassedImages = "passed.html"

    52 KPageForFailedAtImages = "failedAt.html"

    53 # HTML file name for a summary page where passed and failed image information is summarised

    54 KPageForSummary = "summary.html"

    55 

    56 # HTML file name for a summary page of the test results for the overnight build results page

    57 KONBSummary = "directgditest_image_results.htm"

    58 

    59 class TestCase:

    60     # Enum of test results

    61     RESULT_PASSED = 1

    62     RESULT_FAILED = 2

    63 

    64     def __init__(self, aName, aResultDir, aLogDir, aOnbDir):

    65         self.name = aName

    66         self.resultTestImages = []

    67         self.resultRefImages = []

    68         self.resultTestImagesAlpha = []

    69         self.resultRefImagesAlpha = []

    70         self.resultTestImagesWithAlphaPair = []

    71         self.resultRefImagesWithAlphaPair = []

    72         self.resultDir = aResultDir

    73         self.logDir = aLogDir

    74         self.onbDir = aOnbDir

    75         self.testPassedNumber = 0

    76         self.testFailedNumber = 0

    77         self.testFailedAtNumber = 0

    78         self.passedTestImg = []

    79         self.failedTestImg = []

    80         self.failedAtTestImg = []

    81         self.passedRefImg = []

    82         self.failedRefImg = []

    83         self.failedAtRefImg = []

    84         self.thumbnails = {}

    85 

    86     # Read test images to self.resultRefImages

    87     def readRefImage(self, file):

    88     # @param file A log file to store error messages

    89         refimageDir = os.path.join(self.resultDir, KRefPath)

    90         self.resultRefImages = []

    91         self.resultRefImagesAlpha = []

    92         images = os.listdir(refimageDir)

    93                     

    94         

    95         #find bitmaps describing alpha channel first

    96         for imageFile in images:

    97             if lower(imageFile[-10:]) == KImageTypeAlpha:

    98                 self.resultRefImagesAlpha.append(imageFile)

    99                 

   100         #find all bitmaps in directory but those with no alpha pair

   101         for imageFile in images:

   102             alphapair = False

   103             # only process the image with the defined imageType

   104             if lower(imageFile[-4:]) == KImageType and lower(imageFile[-10:]) != KImageTypeAlpha:

   105                 alpharefimg = self.formatNameAddAlpha(imageFile)

   106                 for imgalph in self.resultRefImagesAlpha:

   107                     if alpharefimg == imgalph:

   108                        alphapair = True

   109                        break

   110                        

   111                 if alphapair == False:

   112                    self.resultRefImages.append(imageFile)

   113                 else:

   114                    self.resultRefImagesWithAlphaPair.append(imageFile)

   115 

   116 

   117         if self.resultRefImages == []:

   118             print("WARNING: No ref images found at " + self.resultDir + KRefPath)

   119             file.write("warning: No ref images found at: " + self.resultDir + KRefPath + "<br>")

   120 

   121     # Read test images to self.resultTestImages

   122     # @param file A log file to store error messages

   123     def readTestImage(self, file):

   124         testimageDir = os.path.join(self.resultDir, KTestPath)

   125         self.resultTestImages = []

   126         self.resultTestImagesAlpha = []

   127         images = os.listdir(testimageDir)

   128         

   129         #find bitmaps describing alpha channel first

   130         for imageFile in images:

   131              if lower(imageFile[-10:]) == KImageTypeAlpha:

   132                 self.resultTestImagesAlpha.append(imageFile)

   133                 

   134         #find all bitmaps in directory but those with no alpha pair

   135         for imageFile in images:

   136             alphapair = False

   137             # only process the image with the defined imageType

   138             if lower(imageFile[-4:]) == KImageType and lower(imageFile[-10:]) != KImageTypeAlpha:

   139                 alphatestimg = self.formatNameAddAlpha(imageFile)

   140                 for imgalph in self.resultTestImagesAlpha:

   141                      if alphatestimg == imgalph:

   142                         alphapair = True

   143                         break

   144                         

   145                 if alphapair == False:

   146                    self.resultTestImages.append(imageFile)

   147                 else:

   148                    self.resultTestImagesWithAlphaPair.append(imageFile)

   149 

   150 

   151         if self.resultTestImages == []:

   152             print("WARNING: No test images found at " + self.resultDir + KTestPath)

   153             file.write("warning: No test images found at: " + self.resultDir + KTestPath + "<br>")

   154 

   155     # Remove the prefix of "Bitgdi_" or "Directgdi_" in the image names

   156     # @param aFileName The image file name

   157     def formatBaseName(self, aFileName):

   158         completeName = os.path.basename(aFileName)

   159         index = completeName.find("_")

   160         return completeName[index + 1:]

   161         

   162     # construct name with -alpha postfix

   163     # @param aFileName The image file name

   164     def formatNameAddAlpha(self,aFileName):

   165         (alphaimgname,alphaimgext) = os.path.splitext(aFileName)

   166         return alphaimgname + "-alpha" + alphaimgext

   167         

   168     # construct name by removing -alpha postfix

   169     # @param aFileName The image file name

   170     def formatNameRemoveAlpha(self,aFileName):

   171         (alphaimgname,alphaimgext) = os.path.splitext(aFileName)

   172         index = alphaimgname.find("-alpha")

   173         if index == -1:

   174            return aFileName

   175         else:

   176            return alphaimgname[:index] + alphaimgext

   177 

   178     def processLine(self, aLine):

   179         return string.split(aLine)

   180 

   181     # Compute the difference of test images against their reference images one by one

   182     # @param file A log file to store error messages

   183     def computeDifferences(self, file):

   184 

   185         # Generate list of threshold mappings - 

   186         # Consisting of base name & threshold values

   187         thresholds = []

   188         lineTuple = []

   189         try:

   190           for line in fileinput.input("thresholds.txt"):

   191               lineTuple = self.processLine(line)

   192               if lineTuple:

   193                   try:

   194                       if (lineTuple[0] and (lineTuple[0][0] != '#')):

   195                           thresholds.append(lineTuple[0])

   196                           thresholds.append(lineTuple[1])

   197                           print "Found threshold exception: ", lineTuple[0], " ", lineTuple[1]

   198                   except Exception:

   199                       pass

   200         except Exception:

   201             pass

   202 

   203         # Create file to recieve information on any test failures.

   204         # Format:

   205         # <file name> <pyramid diff failure value>\n

   206         failuresFile = open ( 'testFailures.txt', 'w' ) 

   207 

   208         # loop through test images

   209         for imgFile in self.resultTestImages:

   210             

   211             # get the test image file basename

   212             #baseName_test = os.path.basename(imgFile)

   213             baseName_test = self.formatBaseName(imgFile)

   214 

   215             # only process the image with the defined imageType

   216             if lower(baseName_test[-4:]) == KImageType:

   217 

   218                 # look through reference images

   219                 for refFile in self.resultRefImages:

   220 

   221                     # get the reference image file basename 

   222                     #baseName_ref = os.path.basename(refFile)

   223                     baseName_ref = self.formatBaseName(refFile)

   224 

   225                     # only process the image with the defined imageType

   226                     if lower(baseName_ref[-4:]) == KImageType:

   227 

   228                         # match the file names of a test image with its reference image

   229                         if baseName_test == baseName_ref:

   230 

   231                             # create a RefImage and compute difference between test and reference images

   232                             test = RefImage(refFile, imgFile, self.resultDir, KDiffImageMidName)

   233                             test.computeDifferences(file)

   234                             test.makeDiffImages(self.resultDir)

   235                             

   236                             thresholdValue = -1

   237                             baseCaseName = baseName_test[:-4]

   238                             try:

   239                                 thresholdValue = thresholds[thresholds.index(baseCaseName) + 1]

   240                             except Exception:

   241                                 pass

   242                             

   243                             test.printResult(file, int(thresholdValue))

   244                             result = test.getResult(file, int(thresholdValue))

   245 

   246                             pyramidValue = test.getPyramidResultValue ();

   247                             if result == True:

   248                                 self.testPassedNumber = self.testPassedNumber + 1

   249                                 self.passedTestImg.append((imgFile, pyramidValue))

   250                                 self.passedRefImg.append((refFile, pyramidValue))

   251                                 

   252                                 # If the test passes but is above and independant threshold

   253                                 # record the test and reference image names for report generation.

   254                                 if (pyramidValue > FAILURE_THRESHOLD_MONITOR):

   255                                     self.failedAtTestImg.append((imgFile, pyramidValue))

   256                                     self.failedAtRefImg.append((refFile, pyramidValue))

   257                                     self.testFailedAtNumber = self.testFailedAtNumber + 1

   258                                     

   259                             else:

   260                                 # Output failure information to failures file.

   261                                 strList = []

   262                                 strList.append(baseCaseName)

   263                                 strList.append(`test.pyramidValue()`)

   264                                 strList.append('\n')

   265                                 failuresFile.write(' '.join(strList))

   266                                 print "Test image failure: ", ' '.join(strList)

   267                                 self.testFailedNumber = self.testFailedNumber + 1

   268                                 self.failedTestImg.append((imgFile, pyramidValue))

   269                                 self.failedRefImg.append((refFile, pyramidValue))

   270         

   271         # loop through test images with alpha pair

   272         for imgFile in self.resultTestImagesWithAlphaPair:

   273             

   274             # get the test image file basename

   275             #baseName_test = os.path.basename(imgFile)

   276             baseName_test = self.formatBaseName(imgFile)

   277 

   278             # only process the image with the defined imageType

   279             if lower(baseName_test[-4:]) == KImageType:

   280 

   281                 # look through reference images

   282                 for refFile in self.resultRefImagesWithAlphaPair:

   283 

   284                     # get the reference image file basename 

   285                     #baseName_ref = os.path.basename(refFile)

   286                     baseName_ref = self.formatBaseName(refFile)

   287 

   288                     # only process the image with the defined imageType

   289                     if lower(baseName_ref[-4:]) == KImageType:

   290 

   291                         # match the file names of a test image with its reference image

   292                         if baseName_test == baseName_ref:

   293 

   294                             # create a RefImage and compute difference between test and reference images

   295                             test = RefImage(refFile, imgFile, self.resultDir, KDiffImageMidName)

   296                             test.computeDifferences(file)

   297                             test.makeDiffImages(self.resultDir)

   298 

   299                             thresholdValue = -1

   300                             baseCaseName = baseName_test[:-4]

   301                             try:

   302                                 thresholdValue = thresholds[thresholds.index(baseCaseName) + 1]

   303                             except Exception:

   304                                 pass

   305 

   306                             test.printResult(file, int(thresholdValue))

   307                             result = test.getResult(file, int(thresholdValue))

   308                             

   309                             alpharefimg = self.formatNameAddAlpha(refFile)

   310                             for imgrefalpha in self.resultRefImagesAlpha:

   311                                 if alpharefimg == imgrefalpha:

   312                                    for imgtestalpha in self.resultTestImagesAlpha:

   313                                         if self.formatBaseName(imgrefalpha) == self.formatBaseName(imgtestalpha):

   314                                             testalpha = RefImage(imgrefalpha, imgtestalpha, self.resultDir, KDiffImageMidName)

   315                                             testalpha.computeDifferences(file)

   316                                             testalpha.makeDiffImages(self.resultDir)

   317                                             testalpha.printResult(file, int(thresholdValue))

   318                                             resultalpha = testalpha.getResult(file, int(thresholdValue))

   319                                             if resultalpha == False:

   320                                                result = False

   321                                             break

   322                                                

   323                             pyramidValue = test.getPyramidResultValue ();

   324                             if result == True:

   325                                 self.testPassedNumber = self.testPassedNumber + 1

   326                                 self.passedTestImg.append((imgFile, pyramidValue))

   327                                 self.passedRefImg.append((refFile, pyramidValue))

   328 

   329                                 # If the test passes but is above and independant threshold

   330                                 # record the test and reference image names for report generation.

   331                                 if (pyramidValue > FAILURE_THRESHOLD_MONITOR):

   332                                     self.failedAtTestImg.append((imgFile, pyramidValue))

   333                                     self.failedAtRefImg.append((refFile, pyramidValue))

   334                                     self.testFailedAtNumber = self.testFailedAtNumber + 1

   335 

   336                             else:

   337                                  # Output failure information to failures file.

   338                                 strList = []

   339                                 strList.append(baseCaseName)

   340                                 strList.append(`test.pyramidValue()`)

   341                                 strList.append('\n')

   342                                 failuresFile.write(' '.join(strList))

   343                                 print "Test image alpha-pair failure: ", ' '.join(strList)

   344                                 self.testFailedNumber = self.testFailedNumber + 1

   345                                 self.failedTestImg.append((imgFile, pyramidValue))

   346                                 self.failedRefImg.append((refFile, pyramidValue))

   347                                 

   348         failuresFile.close()

   349         print "All cases: ", self.testPassedNumber + self.testFailedNumber

   350         print "Failed cases: ", self.testFailedNumber

   351         print "Passed cases: ", self.testPassedNumber

   352         print 'Failed at threshold %d cases: %d' % (FAILURE_THRESHOLD_MONITOR, self.testFailedAtNumber)

   353 

   354         # Generate a report of the test results                        

   355         self.generateReport(os.path.join(self.resultDir, KTestPath), os.path.join(self.resultDir, KRefPath), self.resultDir)

   356 

   357     # Generate a report of test results

   358     # @param aTestDir The directory of the test images

   359     # @param aRefDir The directory of the reference images

   360     # @param diffDir The directory of the diff images

   361     def generateReport(self, aTestDir, aRefDir, aDiffDir):

   362         # Write a main index page of the test results

   363         report = file(self.logDir + KIndexPageName, "wt")

   364         writeHtmlHeader(report, self.testFailedNumber, self.testPassedNumber, self.testFailedAtNumber, FAILURE_THRESHOLD_MONITOR)

   365         writeHtmlFooter(report)

   366         report.close()

   367         

   368         #sort tables in alphabetical order

   369         self.failedTestImg.sort()

   370         self.failedRefImg.sort()

   371         self.passedTestImg.sort()

   372         self.passedRefImg.sort()

   373         self.failedAtTestImg.sort()

   374         self.failedAtRefImg.sort()

   375         

   376         summary = file(self.logDir + KPageForSummary, "wt")

   377         writeFailedDetails(summary, self.failedTestImg, self.failedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir)

   378         writePassedDetails(summary, self.passedTestImg, self.passedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir)                

   379         writeFailedAtDetails(summary, self.failedAtTestImg, self.failedAtRefImg, aTestDir, aRefDir, aDiffDir, self.logDir, FAILURE_THRESHOLD_MONITOR) 

   380         

   381         # Write a html page of the failed images

   382         failedDetail = file(self.logDir + KPageForFailedImages, "wt")

   383         writeSimpleHtmlHeader(failedDetail)

   384         writeFailedDetails(failedDetail, self.failedTestImg, self.failedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir)

   385         writeHtmlFooter(failedDetail)

   386         failedDetail.close()

   387         

   388         # Write a html page of the passed images

   389         passedDetail = file(self.logDir + KPageForPassedImages, "wt")

   390         writeSimpleHtmlHeader(passedDetail)

   391         writePassedDetails(passedDetail, self.passedTestImg, self.passedRefImg, aTestDir, aRefDir, aDiffDir, self.logDir)

   392         writeHtmlFooter(passedDetail)

   393         passedDetail.close()

   394 

   395         # Write a html page of the failed at <thresholdvalue> images.

   396         failedAtDetail = file(self.logDir + KPageForFailedAtImages, "wt")

   397         writeSimpleHtmlHeader(failedAtDetail)

   398         writeFailedAtDetails(failedAtDetail, self.failedAtTestImg, self.failedAtRefImg, aTestDir, aRefDir, aDiffDir, self.logDir, FAILURE_THRESHOLD_MONITOR)

   399         writeHtmlFooter(failedAtDetail)

   400         failedAtDetail.close()    

   401         

   402         # Write a summary page for the overnight build results

   403         onb_report = file(self.onbDir + KONBSummary, "wt")

   404         writeHtmlONB(onb_report, self.testFailedNumber, self.testPassedNumber, self.testFailedAtNumber, FAILURE_THRESHOLD_MONITOR, KIndexPageName) 

   405         onb_report.close()