1 /****************************************************************************

     2 **

     3 ** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).

     4 ** All rights reserved.

     5 ** Contact: Nokia Corporation (qt-info@nokia.com)

     6 **

     7 ** This file is part of the Qt Assistant of the Qt Toolkit.

     8 **

     9 ** $QT_BEGIN_LICENSE:LGPL$

    10 ** No Commercial Usage

    11 ** This file contains pre-release code and may not be distributed.

    12 ** You may use this file in accordance with the terms and conditions

    13 ** contained in the Technology Preview License Agreement accompanying

    14 ** this package.

    15 **

    16 ** GNU Lesser General Public License Usage

    17 ** Alternatively, this file may be used under the terms of the GNU Lesser

    18 ** General Public License version 2.1 as published by the Free Software

    19 ** Foundation and appearing in the file LICENSE.LGPL included in the

    20 ** packaging of this file.  Please review the following information to

    21 ** ensure the GNU Lesser General Public License version 2.1 requirements

    22 ** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

    23 **

    24 ** In addition, as a special exception, Nokia gives you certain additional

    25 ** rights.  These rights are described in the Nokia Qt LGPL Exception

    26 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

    27 **

    28 ** If you have questions regarding the use of this file, please contact

    29 ** Nokia at qt-info@nokia.com.

    30 **

    31 **

    32 **

    33 **

    34 **

    35 **

    36 **

    37 **

    38 ** $QT_END_LICENSE$

    39 **

    40 ****************************************************************************/

    41 

    42 #include "qhelpenginecore.h"

    43 #include "qhelp_global.h"

    44 #include "fulltextsearch/qhits_p.h"

    45 #include "fulltextsearch/qquery_p.h"

    46 #include "fulltextsearch/qanalyzer_p.h"

    47 #include "fulltextsearch/qdocument_p.h"

    48 #include "fulltextsearch/qsearchable_p.h"

    49 #include "fulltextsearch/qindexreader_p.h"

    50 #include "fulltextsearch/qindexwriter_p.h"

    51 #include "qhelpsearchindexwriter_clucene_p.h"

    52 

    53 #include <QtCore/QDir>

    54 #include <QtCore/QString>

    55 #include <QtCore/QFileInfo>

    56 #include <QtCore/QTextCodec>

    57 #include <QtCore/QTextStream>

    58 

    59 #include <QtNetwork/QLocalSocket>

    60 #include <QtNetwork/QLocalServer>

    61 

    62 #include "private/qfunctions_p.h"

    63 

    64 QT_BEGIN_NAMESPACE

    65 

    66 namespace fulltextsearch {

    67 namespace clucene {

    68 

    69 // taken from qtexthtmlparser

    70 static const struct QTextHtmlEntity

    71 {

    72     const char *name;

    73     quint16 code;

    74 } entities[] = {

    75     { "AElig", 0x00c6 },

    76     { "AMP", 38 },

    77     { "Aacute", 0x00c1 },

    78     { "Acirc", 0x00c2 },

    79     { "Agrave", 0x00c0 },

    80     { "Alpha", 0x0391 },

    81     { "Aring", 0x00c5 },

    82     { "Atilde", 0x00c3 },

    83     { "Auml", 0x00c4 },

    84     { "Beta", 0x0392 },

    85     { "Ccedil", 0x00c7 },

    86     { "Chi", 0x03a7 },

    87     { "Dagger", 0x2021 },

    88     { "Delta", 0x0394 },

    89     { "ETH", 0x00d0 },

    90     { "Eacute", 0x00c9 },

    91     { "Ecirc", 0x00ca },

    92     { "Egrave", 0x00c8 },

    93     { "Epsilon", 0x0395 },

    94     { "Eta", 0x0397 },

    95     { "Euml", 0x00cb },

    96     { "GT", 62 },

    97     { "Gamma", 0x0393 },

    98     { "Iacute", 0x00cd },

    99     { "Icirc", 0x00ce },

   100     { "Igrave", 0x00cc },

   101     { "Iota", 0x0399 },

   102     { "Iuml", 0x00cf },

   103     { "Kappa", 0x039a },

   104     { "LT", 60 },

   105     { "Lambda", 0x039b },

   106     { "Mu", 0x039c },

   107     { "Ntilde", 0x00d1 },

   108     { "Nu", 0x039d },

   109     { "OElig", 0x0152 },

   110     { "Oacute", 0x00d3 },

   111     { "Ocirc", 0x00d4 },

   112     { "Ograve", 0x00d2 },

   113     { "Omega", 0x03a9 },

   114     { "Omicron", 0x039f },

   115     { "Oslash", 0x00d8 },

   116     { "Otilde", 0x00d5 },

   117     { "Ouml", 0x00d6 },

   118     { "Phi", 0x03a6 },

   119     { "Pi", 0x03a0 },

   120     { "Prime", 0x2033 },

   121     { "Psi", 0x03a8 },

   122     { "QUOT", 34 },

   123     { "Rho", 0x03a1 },

   124     { "Scaron", 0x0160 },

   125     { "Sigma", 0x03a3 },

   126     { "THORN", 0x00de },

   127     { "Tau", 0x03a4 },

   128     { "Theta", 0x0398 },

   129     { "Uacute", 0x00da },

   130     { "Ucirc", 0x00db },

   131     { "Ugrave", 0x00d9 },

   132     { "Upsilon", 0x03a5 },

   133     { "Uuml", 0x00dc },

   134     { "Xi", 0x039e },

   135     { "Yacute", 0x00dd },

   136     { "Yuml", 0x0178 },

   137     { "Zeta", 0x0396 },

   138     { "aacute", 0x00e1 },

   139     { "acirc", 0x00e2 },

   140     { "acute", 0x00b4 },

   141     { "aelig", 0x00e6 },

   142     { "agrave", 0x00e0 },

   143     { "alefsym", 0x2135 },

   144     { "alpha", 0x03b1 },

   145     { "amp", 38 },

   146     { "and", 0x22a5 },

   147     { "ang", 0x2220 },

   148     { "apos", 0x0027 },

   149     { "aring", 0x00e5 },

   150     { "asymp", 0x2248 },

   151     { "atilde", 0x00e3 },

   152     { "auml", 0x00e4 },

   153     { "bdquo", 0x201e },

   154     { "beta", 0x03b2 },

   155     { "brvbar", 0x00a6 },

   156     { "bull", 0x2022 },

   157     { "cap", 0x2229 },

   158     { "ccedil", 0x00e7 },

   159     { "cedil", 0x00b8 },

   160     { "cent", 0x00a2 },

   161     { "chi", 0x03c7 },

   162     { "circ", 0x02c6 },

   163     { "clubs", 0x2663 },

   164     { "cong", 0x2245 },

   165     { "copy", 0x00a9 },

   166     { "crarr", 0x21b5 },

   167     { "cup", 0x222a },

   168     { "curren", 0x00a4 },

   169     { "dArr", 0x21d3 },

   170     { "dagger", 0x2020 },

   171     { "darr", 0x2193 },

   172     { "deg", 0x00b0 },

   173     { "delta", 0x03b4 },

   174     { "diams", 0x2666 },

   175     { "divide", 0x00f7 },

   176     { "eacute", 0x00e9 },

   177     { "ecirc", 0x00ea },

   178     { "egrave", 0x00e8 },

   179     { "empty", 0x2205 },

   180     { "emsp", 0x2003 },

   181     { "ensp", 0x2002 },

   182     { "epsilon", 0x03b5 },

   183     { "equiv", 0x2261 },

   184     { "eta", 0x03b7 },

   185     { "eth", 0x00f0 },

   186     { "euml", 0x00eb },

   187     { "euro", 0x20ac },

   188     { "exist", 0x2203 },

   189     { "fnof", 0x0192 },

   190     { "forall", 0x2200 },

   191     { "frac12", 0x00bd },

   192     { "frac14", 0x00bc },

   193     { "frac34", 0x00be },

   194     { "frasl", 0x2044 },

   195     { "gamma", 0x03b3 },

   196     { "ge", 0x2265 },

   197     { "gt", 62 },

   198     { "hArr", 0x21d4 },

   199     { "harr", 0x2194 },

   200     { "hearts", 0x2665 },

   201     { "hellip", 0x2026 },

   202     { "iacute", 0x00ed },

   203     { "icirc", 0x00ee },

   204     { "iexcl", 0x00a1 },

   205     { "igrave", 0x00ec },

   206     { "image", 0x2111 },

   207     { "infin", 0x221e },

   208     { "int", 0x222b },

   209     { "iota", 0x03b9 },

   210     { "iquest", 0x00bf },

   211     { "isin", 0x2208 },

   212     { "iuml", 0x00ef },

   213     { "kappa", 0x03ba },

   214     { "lArr", 0x21d0 },

   215     { "lambda", 0x03bb },

   216     { "lang", 0x2329 },

   217     { "laquo", 0x00ab },

   218     { "larr", 0x2190 },

   219     { "lceil", 0x2308 },

   220     { "ldquo", 0x201c },

   221     { "le", 0x2264 },

   222     { "lfloor", 0x230a },

   223     { "lowast", 0x2217 },

   224     { "loz", 0x25ca },

   225     { "lrm", 0x200e },

   226     { "lsaquo", 0x2039 },

   227     { "lsquo", 0x2018 },

   228     { "lt", 60 },

   229     { "macr", 0x00af },

   230     { "mdash", 0x2014 },

   231     { "micro", 0x00b5 },

   232     { "middot", 0x00b7 },

   233     { "minus", 0x2212 },

   234     { "mu", 0x03bc },

   235     { "nabla", 0x2207 },

   236     { "nbsp", 0x00a0 },

   237     { "ndash", 0x2013 },

   238     { "ne", 0x2260 },

   239     { "ni", 0x220b },

   240     { "not", 0x00ac },

   241     { "notin", 0x2209 },

   242     { "nsub", 0x2284 },

   243     { "ntilde", 0x00f1 },

   244     { "nu", 0x03bd },

   245     { "oacute", 0x00f3 },

   246     { "ocirc", 0x00f4 },

   247     { "oelig", 0x0153 },

   248     { "ograve", 0x00f2 },

   249     { "oline", 0x203e },

   250     { "omega", 0x03c9 },

   251     { "omicron", 0x03bf },

   252     { "oplus", 0x2295 },

   253     { "or", 0x22a6 },

   254     { "ordf", 0x00aa },

   255     { "ordm", 0x00ba },

   256     { "oslash", 0x00f8 },

   257     { "otilde", 0x00f5 },

   258     { "otimes", 0x2297 },

   259     { "ouml", 0x00f6 },

   260     { "para", 0x00b6 },

   261     { "part", 0x2202 },

   262     { "percnt", 0x0025 },

   263     { "permil", 0x2030 },

   264     { "perp", 0x22a5 },

   265     { "phi", 0x03c6 },

   266     { "pi", 0x03c0 },

   267     { "piv", 0x03d6 },

   268     { "plusmn", 0x00b1 },

   269     { "pound", 0x00a3 },

   270     { "prime", 0x2032 },

   271     { "prod", 0x220f },

   272     { "prop", 0x221d },

   273     { "psi", 0x03c8 },

   274     { "quot", 34 },

   275     { "rArr", 0x21d2 },

   276     { "radic", 0x221a },

   277     { "rang", 0x232a },

   278     { "raquo", 0x00bb },

   279     { "rarr", 0x2192 },

   280     { "rceil", 0x2309 },

   281     { "rdquo", 0x201d },

   282     { "real", 0x211c },

   283     { "reg", 0x00ae },

   284     { "rfloor", 0x230b },

   285     { "rho", 0x03c1 },

   286     { "rlm", 0x200f },

   287     { "rsaquo", 0x203a },

   288     { "rsquo", 0x2019 },

   289     { "sbquo", 0x201a },

   290     { "scaron", 0x0161 },

   291     { "sdot", 0x22c5 },

   292     { "sect", 0x00a7 },

   293     { "shy", 0x00ad },

   294     { "sigma", 0x03c3 },

   295     { "sigmaf", 0x03c2 },

   296     { "sim", 0x223c },

   297     { "spades", 0x2660 },

   298     { "sub", 0x2282 },

   299     { "sube", 0x2286 },

   300     { "sum", 0x2211 },

   301     { "sup", 0x2283 },

   302     { "sup1", 0x00b9 },

   303     { "sup2", 0x00b2 },

   304     { "sup3", 0x00b3 },

   305     { "supe", 0x2287 },

   306     { "szlig", 0x00df },

   307     { "tau", 0x03c4 },

   308     { "there4", 0x2234 },

   309     { "theta", 0x03b8 },

   310     { "thetasym", 0x03d1 },

   311     { "thinsp", 0x2009 },

   312     { "thorn", 0x00fe },

   313     { "tilde", 0x02dc },

   314     { "times", 0x00d7 },

   315     { "trade", 0x2122 },

   316     { "uArr", 0x21d1 },

   317     { "uacute", 0x00fa },

   318     { "uarr", 0x2191 },

   319     { "ucirc", 0x00fb },

   320     { "ugrave", 0x00f9 },

   321     { "uml", 0x00a8 },

   322     { "upsih", 0x03d2 },

   323     { "upsilon", 0x03c5 },

   324     { "uuml", 0x00fc },

   325     { "weierp", 0x2118 },

   326     { "xi", 0x03be },

   327     { "yacute", 0x00fd },

   328     { "yen", 0x00a5 },

   329     { "yuml", 0x00ff },

   330     { "zeta", 0x03b6 },

   331     { "zwj", 0x200d },

   332     { "zwnj", 0x200c }

   333 };

   334 

   335 Q_STATIC_GLOBAL_OPERATOR bool operator<(const QString &entityStr, const QTextHtmlEntity &entity)

   336 {

   337     return entityStr < QLatin1String(entity.name);

   338 }

   339 

   340 Q_STATIC_GLOBAL_OPERATOR bool operator<(const QTextHtmlEntity &entity, const QString &entityStr)

   341 {

   342     return QLatin1String(entity.name) < entityStr;

   343 }

   344 

   345 static QChar resolveEntity(const QString &entity)

   346 {

   347     const QTextHtmlEntity *start = &entities[0];

   348     const QTextHtmlEntity *end = &entities[(sizeof(entities) / sizeof(entities[0]))];

   349     const QTextHtmlEntity *e = qBinaryFind(start, end, entity);

   350     if (e == end)

   351         return QChar();

   352     return e->code;

   353 }

   354 

   355 static const uint latin1Extended[0xA0 - 0x80] = {

   356     0x20ac, // 0x80

   357     0x0081, // 0x81 direct mapping

   358     0x201a, // 0x82

   359     0x0192, // 0x83

   360     0x201e, // 0x84

   361     0x2026, // 0x85

   362     0x2020, // 0x86

   363     0x2021, // 0x87

   364     0x02C6, // 0x88

   365     0x2030, // 0x89

   366     0x0160, // 0x8A

   367     0x2039, // 0x8B

   368     0x0152, // 0x8C

   369     0x008D, // 0x8D direct mapping

   370     0x017D, // 0x8E

   371     0x008F, // 0x8F directmapping

   372     0x0090, // 0x90 directmapping

   373     0x2018, // 0x91

   374     0x2019, // 0x92

   375     0x201C, // 0x93

   376     0X201D, // 0x94

   377     0x2022, // 0x95

   378     0x2013, // 0x96

   379     0x2014, // 0x97

   380     0x02DC, // 0x98

   381     0x2122, // 0x99

   382     0x0161, // 0x9A

   383     0x203A, // 0x9B

   384     0x0153, // 0x9C

   385     0x009D, // 0x9D direct mapping

   386     0x017E, // 0x9E

   387     0x0178  // 0x9F

   388 };

   389 // end taken from qtexthtmlparser

   390 

   391 class DocumentHelper

   392 {

   393 public:

   394     DocumentHelper(const QString &fileName, const QByteArray &data)

   395         : fileName(fileName) , data(readData(data)) {}

   396     ~DocumentHelper() {}

   397 

   398     bool addFieldsToDocument(QCLuceneDocument *document,

   399         const QString &namespaceName, const QString &attributes = QString())

   400     {

   401         if (!document)

   402             return false;

   403 

   404         if(!data.isEmpty()) {

   405             QString parsedData = parseData();

   406             QString parsedTitle = QHelpGlobal::documentTitle(data);

   407 

   408             if(!parsedData.isEmpty()) {

   409                 document->add(new QCLuceneField(QLatin1String("content"),

   410                     parsedData,QCLuceneField::INDEX_TOKENIZED));

   411                 document->add(new QCLuceneField(QLatin1String("path"), fileName,

   412                     QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));

   413                 document->add(new QCLuceneField(QLatin1String("title"), parsedTitle,

   414                     QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));

   415                 document->add(new QCLuceneField(QLatin1String("titleTokenized"), parsedTitle,

   416                     QCLuceneField::STORE_YES | QCLuceneField::INDEX_TOKENIZED));

   417                 document->add(new QCLuceneField(QLatin1String("namespace"), namespaceName,

   418                     QCLuceneField::STORE_YES | QCLuceneField::INDEX_UNTOKENIZED));

   419                 document->add(new QCLuceneField(QLatin1String("attribute"), attributes,

   420                     QCLuceneField::STORE_YES | QCLuceneField::INDEX_TOKENIZED));

   421                 return true;

   422             }

   423         }

   424 

   425         return false;

   426     }

   427 

   428 private:

   429     QString readData(const QByteArray &data)

   430     {

   431         QTextStream textStream(data);

   432         const QByteArray &codec = QHelpGlobal::codecFromData(data).toLatin1();

   433         textStream.setCodec(QTextCodec::codecForName(codec.constData()));

   434 

   435         QString stream = textStream.readAll();

   436         if (stream.isNull() || stream.isEmpty())

   437             return QString();

   438 

   439         return stream;

   440     }

   441 

   442     QString parseData() const

   443     {

   444         const int length = data.length();

   445         const QChar *buf = data.unicode();

   446 

   447         QString parsedContent;

   448         parsedContent.reserve(length);

   449 

   450         bool valid = true;

   451         int j = 0, count = 0;

   452 

   453         QChar c;

   454         while (j < length) {

   455             c = buf[j++];

   456             if (c == QLatin1Char('<') || c == QLatin1Char('&')) {

   457                 if (count > 1 && c != QLatin1Char('&'))

   458                     parsedContent.append(QLatin1Char(' '));

   459                 else if (c == QLatin1Char('&')) {

   460                     // Note: this will modify the counter j, in case we sucessful parsed the entity

   461                     //       we will have modified the counter to stay 1 before the closing ';', so

   462                     //       the following if condition will be met with if (c == QLatin1Char(';'))

   463                     parsedContent.append(parseEntity(length, buf, j));

   464                 }

   465 

   466                 count = 0;

   467                 valid = false;

   468                 continue;

   469             }

   470             if ((c == QLatin1Char('>') || c == QLatin1Char(';')) && !valid) {

   471                 valid = true;

   472                 continue;

   473             }

   474             if (!valid)

   475                 continue;

   476 

   477             if (c.isLetterOrNumber() || c.isPrint()) {

   478                 ++count;

   479                 parsedContent.append(c.toLower());

   480             } else {

   481                 if (count > 1)

   482                     parsedContent.append(QLatin1Char(' '));

   483                 count = 0;

   484             }

   485         }

   486 

   487         return parsedContent;

   488     }

   489 

   490     // taken from qtexthtmlparser

   491     // parses an entity after "&", and returns it

   492     QString parseEntity(int len, const QChar *buf, int &pos) const

   493     {

   494         int recover = pos;

   495         QString entity;

   496         while (pos < len) {

   497             QChar c = buf[pos++];

   498             if (c.isSpace() || pos - recover > 9) {

   499                 goto error;

   500             }

   501             if (c == QLatin1Char(';')) {

   502                 pos--;

   503                 break;

   504             }

   505             entity += c;

   506         }

   507         {

   508             QChar resolved = resolveEntity(entity);

   509             if (!resolved.isNull())

   510                 return QString(resolved);

   511         }

   512         if (entity.length() > 1 && entity.at(0) == QLatin1Char('#')) {

   513             entity.remove(0, 1); // removing leading #

   514 

   515             int base = 10;

   516             bool ok = false;

   517 

   518             if (entity.at(0).toLower() == QLatin1Char('x')) { // hex entity?

   519                 entity.remove(0, 1);

   520                 base = 16;

   521             }

   522 

   523             uint uc = entity.toUInt(&ok, base);

   524             if (ok) {

   525                 if (uc >= 0x80  && uc < 0x80 + (sizeof(latin1Extended) / sizeof(latin1Extended[0])))

   526                     uc = latin1Extended[uc - 0x80]; // windows latin 1 extended

   527                 QString str;

   528                 if (uc > 0xffff) {

   529                     // surrogate pair

   530                     uc -= 0x10000;

   531                     ushort high = uc/0x400 + 0xd800;

   532                     ushort low = uc%0x400 + 0xdc00;

   533                     str.append(QChar(high));

   534                     str.append(QChar(low));

   535                 } else {

   536                     str.append(QChar(uc));

   537                 }

   538                 return str;

   539             }

   540         }

   541     error:

   542         pos = recover;

   543         return QLatin1String(" ");

   544     }

   545     // end taken from qtexthtmlparser

   546 

   547 private:

   548     QString fileName;

   549     QString data;

   550 };

   551 

   552 

   553 QHelpSearchIndexWriter::QHelpSearchIndexWriter()

   554     : QThread(0)

   555     , m_cancel(false)

   556 {

   557     // nothing todo

   558 }

   559 

   560 QHelpSearchIndexWriter::~QHelpSearchIndexWriter()

   561 {

   562     mutex.lock();

   563     this->m_cancel = true;

   564     waitCondition.wakeOne();

   565     mutex.unlock();

   566 

   567     wait();

   568 }

   569 

   570 void QHelpSearchIndexWriter::cancelIndexing()

   571 {

   572     mutex.lock();

   573     this->m_cancel = true;

   574     mutex.unlock();

   575 }

   576 

   577 void QHelpSearchIndexWriter::updateIndex(const QString &collectionFile,

   578     const QString &indexFilesFolder, bool reindex)

   579 {

   580     wait();

   581     mutex.lock();

   582     this->m_cancel = false;

   583     this->m_reindex = reindex;

   584     this->m_collectionFile = collectionFile;

   585     this->m_indexFilesFolder = indexFilesFolder;

   586     mutex.unlock();

   587 

   588     start(QThread::LowestPriority);

   589 }

   590 

   591 void QHelpSearchIndexWriter::optimizeIndex()

   592 {

   593 #if !defined(QT_NO_EXCEPTIONS)

   594     try {

   595 #endif

   596         if (QCLuceneIndexReader::indexExists(m_indexFilesFolder)) {

   597             if (QCLuceneIndexReader::isLocked(m_indexFilesFolder))

   598                 return;

   599 

   600             QCLuceneStandardAnalyzer analyzer;

   601             QCLuceneIndexWriter writer(m_indexFilesFolder, analyzer, false);

   602             writer.optimize();

   603             writer.close();

   604         }

   605 #if !defined(QT_NO_EXCEPTIONS)

   606     } catch (...) {

   607         qWarning("Full Text Search, could not optimize index.");

   608         return;

   609     }

   610 #endif

   611 }

   612 

   613 void QHelpSearchIndexWriter::run()

   614 {

   615 #if !defined(QT_NO_EXCEPTIONS)

   616     try {

   617 #endif

   618         QMutexLocker mutexLocker(&mutex);

   619 

   620         if (m_cancel)

   621             return;

   622 

   623         const bool reindex = this->m_reindex;

   624         const QString collectionFile(this->m_collectionFile);

   625 

   626         mutexLocker.unlock();

   627 

   628         QHelpEngineCore engine(collectionFile, 0);

   629         if (!engine.setupData())

   630             return;

   631 

   632         const QLatin1String key("CluceneIndexedNamespaces");

   633         if (reindex)

   634             engine.setCustomValue(key, QLatin1String(""));

   635 

   636         QMap<QString, QDateTime> indexMap;

   637         const QLatin1String oldKey("CluceneSearchNamespaces");

   638         if (!engine.customValue(oldKey, QString()).isNull()) {

   639             // old style qhc file < 4.4.2, need to convert...

   640             const QStringList indexedNamespaces

   641                 = engine.customValue(oldKey).toString()

   642                   .split(QLatin1String("|"), QString::SkipEmptyParts);

   643             foreach (const QString &nameSpace, indexedNamespaces)

   644                 indexMap.insert(nameSpace, QDateTime());

   645             engine.removeCustomValue(oldKey);

   646         } else {

   647             QDataStream dataStream(engine.customValue(key).toByteArray());

   648             dataStream >> indexMap;

   649         }

   650 

   651         QString indexPath = m_indexFilesFolder;

   652 

   653         QFileInfo fInfo(indexPath);

   654         if (fInfo.exists() && !fInfo.isWritable()) {

   655             qWarning("Full Text Search, could not create index (missing permissions for '%s').",

   656                      qPrintable(indexPath));

   657             return;

   658         }

   659 

   660         emit indexingStarted();

   661 

   662         QCLuceneIndexWriter *writer = 0;

   663         QCLuceneStandardAnalyzer analyzer;

   664         const QStringList registeredDocs = engine.registeredDocumentations();

   665 

   666         QLocalSocket localSocket;

   667         localSocket.connectToServer(QString(QLatin1String("QtAssistant%1"))

   668                                     .arg(QLatin1String(QT_VERSION_STR)));

   669 

   670         QLocalServer localServer;

   671         bool otherInstancesRunning = true;

   672         if (!localSocket.waitForConnected()) {

   673             otherInstancesRunning = false;

   674             localServer.listen(QString(QLatin1String("QtAssistant%1"))

   675                                .arg(QLatin1String(QT_VERSION_STR)));

   676         }

   677 

   678         // check if it's locked, and if the other instance is running

   679         if (!otherInstancesRunning && QCLuceneIndexReader::isLocked(indexPath))

   680             QCLuceneIndexReader::unlock(indexPath);

   681 

   682         if (QCLuceneIndexReader::isLocked(indexPath)) {

   683             // poll unless indexing finished to fake progress

   684             while (QCLuceneIndexReader::isLocked(indexPath)) {

   685                 mutexLocker.relock();

   686                 if (m_cancel)

   687                     break;

   688                 mutexLocker.unlock();

   689                 this->sleep(1);

   690             }

   691             emit indexingFinished();

   692             return;

   693         }

   694 

   695         if (QCLuceneIndexReader::indexExists(indexPath) && !reindex) {

   696             foreach(const QString &namespaceName, registeredDocs) {

   697                 mutexLocker.relock();

   698                 if (m_cancel) {

   699                     emit indexingFinished();

   700                     return;

   701                 }

   702                 mutexLocker.unlock();

   703 

   704                 if (!indexMap.contains(namespaceName)) {

   705                     // make sure we remove some partly indexed stuff

   706                     removeDocuments(indexPath, namespaceName);

   707                 } else {

   708                     QString path = engine.documentationFileName(namespaceName);

   709                     if (indexMap.value(namespaceName)

   710                         < QFileInfo(path).lastModified()) {

   711                         // make sure we remove some outdated indexed stuff

   712                         indexMap.remove(namespaceName);

   713                         removeDocuments(indexPath, namespaceName);

   714                     }

   715 

   716                     if (indexMap.contains(namespaceName)) {

   717                         // make sure we really have content indexed for namespace

   718                         // NOTE: Extra variable just for GCC 3.3.5

   719                         QLatin1String key("namespace");

   720                         QCLuceneTermQuery query(QCLuceneTerm(key, namespaceName));

   721                         QCLuceneIndexSearcher indexSearcher(indexPath);

   722                         QCLuceneHits hits = indexSearcher.search(query);

   723                         if (hits.length() <= 0)

   724                             indexMap.remove(namespaceName);

   725                     }

   726                 }

   727             }

   728             writer = new QCLuceneIndexWriter(indexPath, analyzer, false);

   729         } else {

   730             indexMap.clear();

   731             writer = new QCLuceneIndexWriter(indexPath, analyzer, true);

   732         }

   733 

   734         writer->setMergeFactor(100);

   735         writer->setMinMergeDocs(1000);

   736         writer->setMaxFieldLength(QCLuceneIndexWriter::DEFAULT_MAX_FIELD_LENGTH);

   737 

   738         QStringList namespaces;

   739         foreach(const QString &namespaceName, registeredDocs) {

   740             mutexLocker.relock();

   741             if (m_cancel) {

   742                 closeIndexWriter(writer);

   743                 emit indexingFinished();

   744                 return;

   745             }

   746             mutexLocker.unlock();

   747 

   748             namespaces.append(namespaceName);

   749             if (indexMap.contains(namespaceName))

   750                 continue;

   751 

   752             const QList<QStringList> attributeSets =

   753                     engine.filterAttributeSets(namespaceName);

   754 

   755             if (attributeSets.isEmpty()) {

   756                 const QList<QUrl> docFiles = indexableFiles(&engine, namespaceName,

   757                                                             QStringList());

   758                 if (!addDocuments(docFiles, engine, QStringList(), namespaceName,

   759                                   writer, analyzer))

   760                     break;

   761             } else {

   762                 bool bail = false;

   763                 foreach (const QStringList &attributes, attributeSets) {

   764                     const QList<QUrl> docFiles = indexableFiles(&engine,

   765                                                                 namespaceName, attributes);

   766                     if (!addDocuments(docFiles, engine, attributes, namespaceName,

   767                                       writer, analyzer)) {

   768                         bail = true;

   769                         break;

   770                     }

   771                 }

   772                 if (bail)

   773                     break;

   774             }

   775 

   776             mutexLocker.relock();

   777             if (!m_cancel) {

   778                 QString path(engine.documentationFileName(namespaceName));

   779                 indexMap.insert(namespaceName, QFileInfo(path).lastModified());

   780                 writeIndexMap(engine, indexMap);

   781             }

   782             mutexLocker.unlock();

   783         }

   784 

   785         closeIndexWriter(writer);

   786 

   787         mutexLocker.relock();

   788         if (!m_cancel) {

   789             mutexLocker.unlock();

   790 

   791             QStringList indexedNamespaces = indexMap.keys();

   792             foreach(const QString &namespaceName, indexedNamespaces) {

   793                 mutexLocker.relock();

   794                 if (m_cancel)

   795                     break;

   796                 mutexLocker.unlock();

   797 

   798                 if (!namespaces.contains(namespaceName)) {

   799                     indexMap.remove(namespaceName);

   800                     writeIndexMap(engine, indexMap);

   801                     removeDocuments(indexPath, namespaceName);

   802                 }

   803             }

   804         }

   805 

   806 #if !defined(QT_NO_EXCEPTIONS)

   807     } catch (...) {

   808         qWarning("%s: Failed because of CLucene exception.", Q_FUNC_INFO);

   809     }

   810 #endif

   811 

   812     emit indexingFinished();

   813 }

   814 

   815 bool QHelpSearchIndexWriter::addDocuments(const QList<QUrl> docFiles,

   816     const QHelpEngineCore &engine, const QStringList &attributes,

   817     const QString &namespaceName, QCLuceneIndexWriter *writer,

   818     QCLuceneAnalyzer &analyzer)

   819 {

   820     QMutexLocker locker(&mutex);

   821     const QString attrList = attributes.join(QLatin1String(" "));

   822 

   823     locker.unlock();

   824     foreach(const QUrl &url, docFiles) {

   825         QCLuceneDocument document;

   826         DocumentHelper helper(url.toString(), engine.fileData(url));

   827         if (helper.addFieldsToDocument(&document, namespaceName, attrList)) {

   828 #if !defined(QT_NO_EXCEPTIONS)

   829             try {

   830 #endif

   831                 writer->addDocument(document, analyzer);

   832 #if !defined(QT_NO_EXCEPTIONS)

   833             } catch (...) {

   834                 qWarning("Full Text Search, could not properly add documents.");

   835                 return false;

   836             }

   837 #endif

   838         }

   839         locker.relock();

   840         if (m_cancel)

   841             return false;

   842         locker.unlock();

   843     }

   844     return true;

   845 }

   846 

   847 void QHelpSearchIndexWriter::removeDocuments(const QString &indexPath,

   848     const QString &namespaceName)

   849 {

   850     if (namespaceName.isEmpty() || QCLuceneIndexReader::isLocked(indexPath))

   851         return;

   852 

   853     QCLuceneIndexReader reader = QCLuceneIndexReader::open(indexPath);

   854     reader.deleteDocuments(QCLuceneTerm(QLatin1String("namespace"),

   855         namespaceName));

   856 

   857     reader.close();

   858 }

   859 

   860 bool QHelpSearchIndexWriter::writeIndexMap(QHelpEngineCore &engine,

   861     const QMap<QString, QDateTime> &indexMap)

   862 {

   863     QByteArray bArray;

   864 

   865     QDataStream data(&bArray, QIODevice::ReadWrite);

   866     data << indexMap;

   867 

   868     return engine.setCustomValue(QLatin1String("CluceneIndexedNamespaces"),

   869         bArray);

   870 }

   871 

   872 QList<QUrl> QHelpSearchIndexWriter::indexableFiles(QHelpEngineCore *helpEngine,

   873     const QString &namespaceName, const QStringList &attributes) const

   874 {

   875     QList<QUrl> docFiles = helpEngine->files(namespaceName, attributes,

   876         QLatin1String("html"));

   877     docFiles += helpEngine->files(namespaceName, attributes, QLatin1String("htm"));

   878     docFiles += helpEngine->files(namespaceName, attributes, QLatin1String("txt"));

   879 

   880     return docFiles;

   881 }

   882 

   883 void QHelpSearchIndexWriter::closeIndexWriter(QCLuceneIndexWriter *writer)

   884 {

   885 #if !defined(QT_NO_EXCEPTIONS)

   886     try {

   887 #endif

   888         writer->close();

   889         delete writer;

   890 #if !defined(QT_NO_EXCEPTIONS)

   891     } catch (...) {

   892         qWarning("Full Text Search, could not properly close index writer.");

   893     }

   894 #endif

   895 }

   896 

   897 }   // namespace clucene

   898 }   // namespace fulltextsearch

   899 

   900 QT_END_NAMESPACE