searchengine/cpix/tsrc/cpixunittest/src/documenttest.cpp
changeset 0 671dee74050a
child 3 ae3f1779f6da
equal deleted inserted replaced
-1:000000000000 0:671dee74050a
       
     1 /*
       
     2 * Copyright (c) 2010 Nokia Corporation and/or its subsidiary(-ies).
       
     3 * All rights reserved.
       
     4 * This component and the accompanying materials are made available
       
     5 * under the terms of "Eclipse Public License v1.0"
       
     6 * which accompanies this distribution, and is available
       
     7 * at the URL "http://www.eclipse.org/legal/epl-v10.html".
       
     8 *
       
     9 * Initial Contributors:
       
    10 * Nokia Corporation - initial contribution.
       
    11 *
       
    12 * Contributors:
       
    13 *
       
    14 * Description: 
       
    15 *
       
    16 */
       
    17 
       
    18 #include <wchar.h>
       
    19 #include <stddef.h>
       
    20 
       
    21 #include <iostream>
       
    22 #include <set>
       
    23 #include <string>
       
    24 
       
    25 #include "cpixtools.h"
       
    26 
       
    27 #include "itk.h"
       
    28 
       
    29 #include "cpixidxdb.h"
       
    30 #include "cpixdoc.h"
       
    31 #include "cpixsearch.h"
       
    32 #include "cpixidxdb.h"
       
    33 
       
    34 #include "config.h"
       
    35 #include "testutils.h"
       
    36 #include "setupsentry.h"
       
    37 
       
    38 #define TEST_DOCUMENT_QBASEAPPCLASS "@0:root test document"
       
    39 #define TEST_DOCUMENT_INDEXDB_PATH "c:\\Data\\indexing\\indexdb\\root\\test\\document"
       
    40 
       
    41 #define DOCUID1 "document1" // without boosting this document will never be first result
       
    42 #define DOCUID2 "document2"
       
    43 #define LDOCUID1 L"document1"
       
    44 #define LDOCUID2 L"document2"
       
    45 
       
    46 // Each document will have 2 fields, alpha
       
    47 #define FIELD_ALPHA L"Alpha"
       
    48 
       
    49 
       
    50 #define DOC1CONTENT L"mary had a little lamb its fleece was black as coal"
       
    51 #define DOC2CONTENT L"mary had a little little little lamb its fleece was as white as snow"
       
    52 
       
    53 
       
    54 // The term that will be present in multiple documents.
       
    55 #define SEARCH_TERM L"little"
       
    56 
       
    57 // black is present in doc1, white in doc2. 
       
    58 #define BOOST_SEARCH_TERMS L"black^20 white"
       
    59 
       
    60 #define DEF_BOOST 1.0
       
    61 
       
    62 
       
    63 class DocumentContext : public Itk::ITestContext
       
    64 {
       
    65 private:
       
    66     //
       
    67     // private members
       
    68     //
       
    69     cpix_Analyzer * analyzer_;
       
    70     cpix_QueryParser * queryParser_;
       
    71     cpix_Query * query_;
       
    72     cpix_IdxDb * idxDb_;
       
    73     cpix_Hits * hits_;
       
    74     
       
    75 private:
       
    76   //  Search for the term SEARCH_TERM in field Alpha
       
    77     void executeSearch(Itk::TestMgr * testMgr)
       
    78     {
       
    79         hits_ = cpix_IdxDb_search(idxDb_, query_);
       
    80         
       
    81         int32_t 
       
    82         hits_len = cpix_Hits_length(hits_);
       
    83         
       
    84         ITK_ASSERT(testMgr,
       
    85             hits_len == 2,
       
    86             "wrong amount of documents returned in hits");
       
    87     }
       
    88 
       
    89     void addDocument(Itk::TestMgr  * testMgr,
       
    90                      const wchar_t * docUid,
       
    91                      const wchar_t * fieldValue)
       
    92     {
       
    93         addDocument(testMgr,
       
    94                     docUid,
       
    95                     fieldValue,
       
    96                     DEF_BOOST,
       
    97                     DEF_BOOST);
       
    98     }
       
    99                      
       
   100     
       
   101     
       
   102     void addDocument(Itk::TestMgr  * testMgr,
       
   103                      const wchar_t * docUid,
       
   104                      const wchar_t * fieldValue,
       
   105                      float_t         docBoost,
       
   106                      float_t         fieldBoost) // TODO IMPLEMENT
       
   107     {
       
   108         cpix_Result
       
   109             result;
       
   110         cpix_Document
       
   111             * doc = cpix_Document_create(&result,
       
   112                                          docUid,
       
   113                                          NULL,      // app class
       
   114                                          NULL,      // excerpt
       
   115                                          NULL);     // mime type
       
   116         ITK_ASSERT(testMgr,
       
   117                    cpix_Succeeded(&result),
       
   118                    "Creating document %S failed",
       
   119                    docUid);
       
   120 
       
   121         if (docBoost != DEF_BOOST)
       
   122             {
       
   123                 cpix_Document_setBoost(doc,
       
   124                                        docBoost);
       
   125                 ITK_EXPECT(testMgr,
       
   126                            cpix_Succeeded(doc),
       
   127                            "Could not set doc boost");
       
   128 
       
   129                 float_t
       
   130                     boost = cpix_Document_boost(doc);
       
   131 
       
   132                 ITK_EXPECT(testMgr,
       
   133                            cpix_Succeeded(doc),
       
   134                            "Could not get doc boost");
       
   135                 ITK_EXPECT(testMgr,
       
   136                            boost == docBoost,
       
   137                            "Incorrect doc boost value");
       
   138             }
       
   139 
       
   140         cpix_Field
       
   141             field;
       
   142         cpix_Field_initialize(&field,
       
   143                               FIELD_ALPHA,
       
   144                               fieldValue, 
       
   145                               cpix_STORE_YES |cpix_INDEX_TOKENIZED);
       
   146         if (cpix_Failed(&field))
       
   147             {
       
   148                 cpix_Document_destroy(doc);
       
   149             }
       
   150 
       
   151         ITK_ASSERT(testMgr,
       
   152                    cpix_Succeeded(&field),
       
   153                    "Could not create field");
       
   154 
       
   155         if (fieldBoost != DEF_BOOST)
       
   156             {
       
   157                 cpix_Field_setBoost(&field,
       
   158                                     fieldBoost);
       
   159                 ITK_EXPECT(testMgr,
       
   160                            cpix_Succeeded(&field),
       
   161                            "Could not set field boost");
       
   162 
       
   163                 float_t
       
   164                     boost = cpix_Field_boost(&field);
       
   165 
       
   166                 ITK_EXPECT(testMgr,
       
   167                            cpix_Succeeded(&field),
       
   168                            "Could not get field boost");
       
   169                 ITK_EXPECT(testMgr,
       
   170                            boost == fieldBoost,
       
   171                            "Incorrect field boost value");
       
   172             }
       
   173         
       
   174         cpix_Document_add(doc,
       
   175                           &field);
       
   176         bool
       
   177             succeeded = true;
       
   178         if (cpix_Failed(doc))
       
   179             {
       
   180                 cpix_Document_destroy(doc);
       
   181                 cpix_Field_release(&field);
       
   182                 succeeded = false;
       
   183             }
       
   184 
       
   185         ITK_ASSERT(testMgr,
       
   186                    succeeded,
       
   187                    "Could not add field to document");
       
   188 
       
   189         cpix_IdxDb_add(idxDb_,
       
   190                        doc,
       
   191                        analyzer_);
       
   192 
       
   193         cpix_Document_destroy(doc);
       
   194 
       
   195         ITK_ASSERT(testMgr,
       
   196                    cpix_Succeeded(idxDb_),
       
   197                    "Could not add document to index");
       
   198     }
       
   199 
       
   200 public:
       
   201     
       
   202     //
       
   203     // from ITestContext
       
   204     //
       
   205     virtual void setup() throw (Itk::PanicExc)
       
   206     {
       
   207         cpix_Result
       
   208             result;
       
   209 
       
   210         SetupSentry
       
   211             ss(*this);
       
   212 
       
   213         cpix_IdxDb_dbgScrapAll(&result);
       
   214 
       
   215         if (cpix_Failed(&result))
       
   216         {
       
   217             ITK_PANIC("cpix_IdxDb_dbgScrapAll problem");
       
   218         }
       
   219         
       
   220         analyzer_ = cpix_CreateSimpleAnalyzer(&result);
       
   221         if ( !analyzer_ )
       
   222         {
       
   223             ITK_PANIC("Analyzer could not be created");
       
   224         }
       
   225 
       
   226         queryParser_ = cpix_QueryParser_create(&result,
       
   227                                                 FIELD_ALPHA,
       
   228                                                analyzer_);
       
   229         if (queryParser_ == NULL)
       
   230             {
       
   231                 ITK_PANIC("Could not create query parser");
       
   232             }
       
   233 
       
   234         query_ = cpix_QueryParser_parse(queryParser_,
       
   235                                         SEARCH_TERM);
       
   236         if (cpix_Failed(queryParser_))
       
   237             {
       
   238                 ITK_PANIC("Could not create query parser");
       
   239             }
       
   240 
       
   241         cpix_IdxDb_defineVolume(&result, 
       
   242                                 TEST_DOCUMENT_QBASEAPPCLASS, 
       
   243                                 TEST_DOCUMENT_INDEXDB_PATH);
       
   244         if (cpix_Failed(&result))
       
   245             {
       
   246             ITK_PANIC("Failed to define test index db");
       
   247             }
       
   248 
       
   249         idxDb_ = cpix_IdxDb_openDb(&result,
       
   250                                    TEST_DOCUMENT_QBASEAPPCLASS,
       
   251                                    cpix_IDX_CREATE);
       
   252         if (cpix_Failed(&result))
       
   253             {
       
   254             ITK_PANIC("Failed to open indexDb");
       
   255             }
       
   256 
       
   257         ss.setupComplete();
       
   258     }
       
   259 
       
   260     
       
   261     virtual void tearDown() throw()
       
   262     {
       
   263         cpix_Analyzer_destroy(analyzer_);
       
   264         analyzer_ = NULL;
       
   265         
       
   266         cpix_QueryParser_destroy(queryParser_);
       
   267         queryParser_ = NULL;
       
   268         
       
   269         cpix_Query_destroy(query_);
       
   270         query_ = NULL;
       
   271 
       
   272         cpix_IdxDb_releaseDb(idxDb_);
       
   273         idxDb_ = NULL;
       
   274 
       
   275         cpix_Hits_destroy(hits_);
       
   276         hits_ = NULL;
       
   277     }
       
   278 
       
   279     
       
   280     DocumentContext()
       
   281       : analyzer_(NULL),
       
   282         queryParser_(NULL),
       
   283         query_(NULL),
       
   284         idxDb_(NULL),
       
   285         hits_(NULL)
       
   286     {
       
   287         ;
       
   288     }
       
   289 
       
   290     ~DocumentContext()
       
   291     {
       
   292     }
       
   293 
       
   294     void testNoBoostingFields(Itk::TestMgr * testMgr)
       
   295     {
       
   296         // Don't boost Field Alpha in doc1
       
   297 
       
   298         addDocument(testMgr,
       
   299                     LDOCUID1,
       
   300                     DOC1CONTENT);
       
   301         addDocument(testMgr,
       
   302                     LDOCUID2,
       
   303                     DOC2CONTENT);
       
   304         
       
   305         cpix_IdxDb_flush(idxDb_);
       
   306         ITK_EXPECT(testMgr,
       
   307                    cpix_Succeeded(idxDb_),
       
   308                    "Flushing index has failed");
       
   309 
       
   310         executeSearch(testMgr);
       
   311         //  EXPECTED result is that doc2 first, doc1 second.
       
   312 
       
   313         cpix_Document
       
   314             returnedDoc1;
       
   315 
       
   316         cpix_Hits_doc(hits_, 0, &returnedDoc1);
       
   317 
       
   318         const wchar_t* id = cpix_Document_getFieldValue(&returnedDoc1, 
       
   319                                                         LCPIX_DOCUID_FIELD);
       
   320         if (id)
       
   321         {
       
   322             std::wstring str(id);
       
   323             ITK_ASSERT(testMgr,
       
   324                 str.compare(LDOCUID2) == 0,
       
   325                 "wrong document");
       
   326         }
       
   327         else
       
   328         {
       
   329             ITK_PANIC("failed to get _docuid");
       
   330         }
       
   331 
       
   332         cpix_Document
       
   333             returnedDoc2;
       
   334 
       
   335         cpix_Hits_doc(hits_, 1, &returnedDoc2);
       
   336         id = NULL;
       
   337 
       
   338         id = cpix_Document_getFieldValue(&returnedDoc2, LCPIX_DOCUID_FIELD);
       
   339         if (id)
       
   340         {
       
   341             std::wstring str(id);
       
   342             ITK_ASSERT(testMgr,
       
   343                 str.compare(LDOCUID1) == 0,
       
   344                 "wrong document");
       
   345         }
       
   346         else
       
   347         {
       
   348             ITK_PANIC("failed to get _docuid");
       
   349         }
       
   350     }
       
   351 
       
   352 
       
   353 
       
   354     void testBoostField(Itk::TestMgr * testMgr)
       
   355     {
       
   356         tearDown();
       
   357         setup();
       
   358 
       
   359         addDocument(testMgr,
       
   360                     LDOCUID1,
       
   361                     DOC1CONTENT,
       
   362                     DEF_BOOST,
       
   363                     4.0);
       
   364         addDocument(testMgr,
       
   365                     LDOCUID2,
       
   366                     DOC2CONTENT);
       
   367 
       
   368         cpix_IdxDb_flush(idxDb_);
       
   369         ITK_EXPECT(testMgr,
       
   370                    cpix_Succeeded(idxDb_),
       
   371                    "Flushing index has failed");
       
   372 
       
   373         executeSearch(testMgr);
       
   374         //  EXPECTED result is that doc1 first, doc2 second.
       
   375 
       
   376         cpix_Document
       
   377             returnedDoc1;
       
   378         
       
   379         cpix_Hits_doc(hits_, 0, &returnedDoc1);
       
   380 
       
   381         const wchar_t* id = cpix_Document_getFieldValue(&returnedDoc1, 
       
   382                                                         LCPIX_DOCUID_FIELD);
       
   383         if ( id )
       
   384         {
       
   385             std::wstring str( id );
       
   386             ITK_ASSERT(testMgr,
       
   387                 str.compare(LDOCUID1) == 0,
       
   388                 "wrong document");
       
   389         }
       
   390         else
       
   391         {
       
   392             ITK_PANIC("failed to get _docuid");
       
   393         }
       
   394 
       
   395         cpix_Document
       
   396             returnedDoc2;
       
   397 
       
   398         cpix_Hits_doc(hits_, 1, &returnedDoc2);
       
   399         id = NULL;
       
   400 
       
   401         id = cpix_Document_getFieldValue(&returnedDoc2, LCPIX_DOCUID_FIELD );
       
   402         if ( id )
       
   403         {
       
   404             std::wstring str( id );
       
   405             ITK_ASSERT(testMgr,
       
   406                 str.compare(LDOCUID2) == 0,
       
   407                 "wrong document");
       
   408         }
       
   409         else
       
   410         {
       
   411             ITK_PANIC("failed to get _docuid");
       
   412         }
       
   413     }
       
   414 
       
   415     void testBoostDocument(Itk::TestMgr * testMgr)
       
   416     {
       
   417         tearDown();
       
   418         setup();
       
   419     
       
   420         addDocument(testMgr,
       
   421                     LDOCUID1,
       
   422                     DOC1CONTENT,
       
   423                     2.0,
       
   424                     DEF_BOOST);
       
   425         addDocument(testMgr,
       
   426                     LDOCUID2,
       
   427                     DOC2CONTENT);
       
   428 
       
   429         cpix_IdxDb_flush(idxDb_);
       
   430         ITK_EXPECT(testMgr,
       
   431                    cpix_Succeeded(idxDb_),
       
   432                    "Flushing index has failed");
       
   433 
       
   434         executeSearch(testMgr);
       
   435         //  EXPECTED result is that doc1 first, doc2 second.
       
   436 
       
   437         cpix_Document
       
   438             returnedDoc1;
       
   439         
       
   440         cpix_Hits_doc(hits_, 0, &returnedDoc1);
       
   441 
       
   442         const wchar_t* id = cpix_Document_getFieldValue(&returnedDoc1, 
       
   443                                                         LCPIX_DOCUID_FIELD);
       
   444         if (id)
       
   445         {
       
   446             std::wstring str(id);
       
   447             ITK_ASSERT(testMgr,
       
   448                 str.compare(LDOCUID1) == 0,
       
   449                 "wrong document");
       
   450         }
       
   451         else
       
   452         {
       
   453             ITK_PANIC("failed to get _docuid");
       
   454         }
       
   455 
       
   456         cpix_Document
       
   457             returnedDoc2;
       
   458 
       
   459         cpix_Hits_doc(hits_, 1, &returnedDoc2);
       
   460         id = NULL;
       
   461 
       
   462         id = cpix_Document_getFieldValue(&returnedDoc2, LCPIX_DOCUID_FIELD);
       
   463         if (id)
       
   464         {
       
   465             std::wstring str(id);
       
   466             ITK_ASSERT(testMgr,
       
   467                 str.compare(LDOCUID2) == 0,
       
   468                 "wrong document");
       
   469         }
       
   470         else
       
   471         {
       
   472             ITK_PANIC("failed to get _docuid");
       
   473         }
       
   474     }
       
   475 
       
   476 
       
   477     void testBoostQuery(Itk::TestMgr * testMgr)
       
   478     {
       
   479         tearDown();
       
   480         setup();
       
   481 
       
   482         addDocument(testMgr,
       
   483                     LDOCUID1,
       
   484                     DOC1CONTENT);
       
   485         addDocument(testMgr,
       
   486                     LDOCUID2,
       
   487                     DOC2CONTENT);
       
   488 
       
   489         cpix_IdxDb_flush(idxDb_);
       
   490         ITK_EXPECT(testMgr,
       
   491                    cpix_Succeeded(idxDb_),
       
   492                    "Flushing index has failed");
       
   493 
       
   494         // doc1_ should be the first result given the following query boost.
       
   495         cpix_Query_destroy(query_);
       
   496 
       
   497         query_ = cpix_QueryParser_parse(queryParser_,
       
   498                                         BOOST_SEARCH_TERMS);
       
   499 
       
   500         hits_ = cpix_IdxDb_search(idxDb_, query_);
       
   501       
       
   502         int32_t
       
   503             hits_len = cpix_Hits_length(hits_);
       
   504 
       
   505         ITK_ASSERT(testMgr,
       
   506                   hits_len == 2,
       
   507                   "wrong amount of documents returned in hits");
       
   508 
       
   509         //  EXPECTED result is that doc2 first.
       
   510         cpix_Document
       
   511             returnedDoc1;
       
   512         
       
   513         cpix_Hits_doc(hits_, 0, &returnedDoc1);
       
   514 
       
   515         const wchar_t* id = cpix_Document_getFieldValue(&returnedDoc1, 
       
   516                                                         LCPIX_DOCUID_FIELD);
       
   517         if (id)
       
   518         {
       
   519             std::wstring str(id);
       
   520             ITK_ASSERT(testMgr,
       
   521                 str.compare(LDOCUID1) == 0,
       
   522                 "wrong document");
       
   523         }
       
   524         else
       
   525         {
       
   526             ITK_PANIC("failed to get _docuid");
       
   527         }
       
   528 
       
   529         cpix_Document
       
   530             returnedDoc2;
       
   531 
       
   532         cpix_Hits_doc(hits_, 1, &returnedDoc2);
       
   533         id = NULL;
       
   534 
       
   535         id = cpix_Document_getFieldValue(&returnedDoc2, LCPIX_DOCUID_FIELD);
       
   536         if (id)
       
   537         {
       
   538             std::wstring str(id);
       
   539             ITK_ASSERT(testMgr,
       
   540                 str.compare(LDOCUID2) == 0,
       
   541                 "wrong document");
       
   542         }
       
   543         else
       
   544         {
       
   545             ITK_PANIC("failed to get _docuid");
       
   546         }
       
   547     }
       
   548 };
       
   549 
       
   550 
       
   551 Itk::TesterBase * CreateDocumentTests()
       
   552 {
       
   553     using namespace Itk;
       
   554 
       
   555     DocumentContext
       
   556         * documentContext = new DocumentContext;
       
   557     ContextTester
       
   558         * contextTester = new ContextTester("document",
       
   559                                             documentContext);
       
   560 
       
   561 #define TEST "sanity check no boosts"
       
   562     contextTester->add(TEST,
       
   563             documentContext,
       
   564                        &DocumentContext::testNoBoostingFields);
       
   565 #undef TEST
       
   566 
       
   567 #define TEST "boost field"
       
   568     contextTester->add(TEST,
       
   569             documentContext,
       
   570                        &DocumentContext::testBoostField);
       
   571 #undef TEST
       
   572 
       
   573 #define TEST "boost document"
       
   574     contextTester->add(TEST,
       
   575             documentContext,
       
   576                        &DocumentContext::testBoostDocument);
       
   577 #undef TEST
       
   578 
       
   579 #define TEST "boost query"
       
   580     contextTester->add(TEST,
       
   581             documentContext,
       
   582                        &DocumentContext::testBoostQuery);
       
   583 #undef TEST
       
   584         
       
   585     return contextTester;
       
   586 }