/ Changes On Branch fts5-perf
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch fts5-perf Excluding Merge-Ins

This is equivalent to a diff from 3be336aa89 to b4ac61aeee

2016-01-26
20:19
Performance improvements for fts5, particularly detail=col mode. (check-in: a3d7b8ac53 user: dan tags: trunk)
20:08
Further minor performance improvements and code-size reductions related to fts5 column filters on detail=col tables. (Leaf check-in: b4ac61aeee user: dan tags: fts5-perf)
19:30
Improve the performance of fts5 column filters on detail=col tables. (check-in: 249a2d070c user: dan tags: fts5-perf)
2016-01-23
18:24
Changes to spellfix to try to get it to use stack space instead of heap space in cases where that makes sense. (check-in: dfcebc7393 user: drh tags: trunk)
16:20
Merge trunk changes (including fixes for warnings in fts5) with this branch. (check-in: ceccc9ad78 user: dan tags: fts5-perf)
15:57
Fix some signed/unsigned comparison compiler warnings in fts5. (check-in: 3be336aa89 user: dan tags: trunk)
14:05
Remove an assert() that can be false if compiled with SQLITE_USE_ALLOCA. (check-in: f0a551edf8 user: drh tags: trunk)

Changes to ext/fts5/fts5Int.h.

    25     25   
    26     26   typedef unsigned char  u8;
    27     27   typedef unsigned int   u32;
    28     28   typedef unsigned short u16;
    29     29   typedef sqlite3_int64 i64;
    30     30   typedef sqlite3_uint64 u64;
    31     31   
    32         -#define ArraySize(x) (sizeof(x) / sizeof(x[0]))
           32  +#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0])))
    33     33   
    34     34   #define testcase(x)
    35     35   #define ALWAYS(x) 1
    36     36   #define NEVER(x) 0
    37     37   
    38     38   #define MIN(x,y) (((x) < (y)) ? (x) : (y))
    39     39   #define MAX(x,y) (((x) > (y)) ? (x) : (y))
................................................................................
   311    311   ** Interface to code in fts5_index.c. fts5_index.c contains contains code
   312    312   ** to access the data stored in the %_data table.
   313    313   */
   314    314   
   315    315   typedef struct Fts5Index Fts5Index;
   316    316   typedef struct Fts5IndexIter Fts5IndexIter;
   317    317   
          318  +struct Fts5IndexIter {
          319  +  i64 iRowid;
          320  +  const u8 *pData;
          321  +  int nData;
          322  +};
          323  +
   318    324   /*
   319    325   ** Values used as part of the flags argument passed to IndexQuery().
   320    326   */
   321    327   #define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
   322    328   #define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
   323    329   #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
   324    330   #define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
................................................................................
   378    384   ** The various operations on open token or token prefix iterators opened
   379    385   ** using sqlite3Fts5IndexQuery().
   380    386   */
   381    387   int sqlite3Fts5IterEof(Fts5IndexIter*);
   382    388   int sqlite3Fts5IterNext(Fts5IndexIter*);
   383    389   int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
   384    390   i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
   385         -int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*);
   386         -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf);
   387    391   
   388    392   /*
   389    393   ** Close an iterator opened by sqlite3Fts5IndexQuery().
   390    394   */
   391    395   void sqlite3Fts5IterClose(Fts5IndexIter*);
   392    396   
   393    397   /*
................................................................................
   465    469   
   466    470   int sqlite3Fts5IndexReinit(Fts5Index *p);
   467    471   int sqlite3Fts5IndexOptimize(Fts5Index *p);
   468    472   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
   469    473   
   470    474   int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
   471    475   
   472         -int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
   473         -
   474    476   /*
   475    477   ** End of interface to code in fts5_index.c.
   476    478   **************************************************************************/
   477    479   
   478    480   /**************************************************************************
   479    481   ** Interface to code in fts5_varint.c. 
   480    482   */

Changes to ext/fts5/fts5_aux.c.

   540    540       { "snippet",   0, fts5SnippetFunction, 0 },
   541    541       { "highlight", 0, fts5HighlightFunction, 0 },
   542    542       { "bm25",      0, fts5Bm25Function,    0 },
   543    543     };
   544    544     int rc = SQLITE_OK;             /* Return code */
   545    545     int i;                          /* To iterate through builtin functions */
   546    546   
   547         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
          547  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
   548    548       rc = pApi->xCreateFunction(pApi,
   549    549           aBuiltin[i].zFunc,
   550    550           aBuiltin[i].pUserData,
   551    551           aBuiltin[i].xFunc,
   552    552           aBuiltin[i].xDestroy
   553    553       );
   554    554     }
   555    555   
   556    556     return rc;
   557    557   }
   558    558   
   559    559   

Changes to ext/fts5/fts5_buffer.c.

   318    318     const char *pTerm, int nTerm, 
   319    319     int *pbPresent
   320    320   ){
   321    321     int rc = SQLITE_OK;
   322    322     *pbPresent = 0;
   323    323     if( p ){
   324    324       int i;
   325         -    int hash = 13;
          325  +    u32 hash = 13;
   326    326       Fts5TermsetEntry *pEntry;
   327    327   
   328    328       /* Calculate a hash value for this term. This is the same hash checksum
   329    329       ** used by the fts5_hash.c module. This is not important for correct
   330    330       ** operation of the module, but is necessary to ensure that some tests
   331    331       ** designed to produce hash table collisions really do work.  */
   332    332       for(i=nTerm-1; i>=0; i--){
................................................................................
   335    335       hash = (hash << 3) ^ hash ^ iIdx;
   336    336       hash = hash % ArraySize(p->apHash);
   337    337   
   338    338       for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
   339    339         if( pEntry->iIdx==iIdx 
   340    340             && pEntry->nTerm==nTerm 
   341    341             && memcmp(pEntry->pTerm, pTerm, nTerm)==0 
   342         -        ){
          342  +      ){
   343    343           *pbPresent = 1;
   344    344           break;
   345    345         }
   346    346       }
   347    347   
   348    348       if( pEntry==0 ){
   349    349         pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);

Changes to ext/fts5/fts5_expr.c.

   302    302   ** Argument pTerm must be a synonym iterator.
   303    303   */
   304    304   static int fts5ExprSynonymList(
   305    305     Fts5ExprTerm *pTerm, 
   306    306     int bCollist, 
   307    307     Fts5Colset *pColset,
   308    308     i64 iRowid,
   309         -  int *pbDel,                     /* OUT: Caller should sqlite3_free(*pa) */
          309  +  Fts5Buffer *pBuf,               /* Use this buffer for space if required */
   310    310     u8 **pa, int *pn
   311    311   ){
   312    312     Fts5PoslistReader aStatic[4];
   313    313     Fts5PoslistReader *aIter = aStatic;
   314    314     int nIter = 0;
   315    315     int nAlloc = 4;
   316    316     int rc = SQLITE_OK;
   317    317     Fts5ExprTerm *p;
   318    318   
   319    319     assert( pTerm->pSynonym );
   320    320     for(p=pTerm; p; p=p->pSynonym){
   321    321       Fts5IndexIter *pIter = p->pIter;
   322    322       if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
   323         -      const u8 *a;
   324         -      int n;
   325         -
   326         -      if( bCollist ){
   327         -        rc = sqlite3Fts5IterCollist(pIter, &a, &n);
   328         -      }else{
   329         -        i64 dummy;
   330         -        rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
   331         -      }
   332         -
   333         -      if( rc!=SQLITE_OK ) goto synonym_poslist_out;
   334         -      if( n==0 ) continue;
          323  +      if( pIter->nData==0 ) continue;
   335    324         if( nIter==nAlloc ){
   336    325           int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
   337    326           Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   338    327           if( aNew==0 ){
   339    328             rc = SQLITE_NOMEM;
   340    329             goto synonym_poslist_out;
   341    330           }
   342    331           memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
   343    332           nAlloc = nAlloc*2;
   344    333           if( aIter!=aStatic ) sqlite3_free(aIter);
   345    334           aIter = aNew;
   346    335         }
   347         -      sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]);
          336  +      sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &aIter[nIter]);
   348    337         assert( aIter[nIter].bEof==0 );
   349    338         nIter++;
   350    339       }
   351    340     }
   352    341   
   353         -  assert( *pbDel==0 );
   354    342     if( nIter==1 ){
   355    343       *pa = (u8*)aIter[0].a;
   356    344       *pn = aIter[0].n;
   357    345     }else{
   358    346       Fts5PoslistWriter writer = {0};
   359         -    Fts5Buffer buf = {0,0,0};
   360    347       i64 iPrev = -1;
          348  +    fts5BufferZero(pBuf);
   361    349       while( 1 ){
   362    350         int i;
   363    351         i64 iMin = FTS5_LARGEST_INT64;
   364    352         for(i=0; i<nIter; i++){
   365    353           if( aIter[i].bEof==0 ){
   366    354             if( aIter[i].iPos==iPrev ){
   367    355               if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) continue;
................................................................................
   368    356             }
   369    357             if( aIter[i].iPos<iMin ){
   370    358               iMin = aIter[i].iPos;
   371    359             }
   372    360           }
   373    361         }
   374    362         if( iMin==FTS5_LARGEST_INT64 || rc!=SQLITE_OK ) break;
   375         -      rc = sqlite3Fts5PoslistWriterAppend(&buf, &writer, iMin);
          363  +      rc = sqlite3Fts5PoslistWriterAppend(pBuf, &writer, iMin);
   376    364         iPrev = iMin;
   377    365       }
   378         -    if( rc ){
   379         -      sqlite3_free(buf.p);
   380         -    }else{
   381         -      *pa = buf.p;
   382         -      *pn = buf.n;
   383         -      *pbDel = 1;
          366  +    if( rc==SQLITE_OK ){
          367  +      *pa = pBuf->p;
          368  +      *pn = pBuf->n;
   384    369       }
   385    370     }
   386    371   
   387    372    synonym_poslist_out:
   388    373     if( aIter!=aStatic ) sqlite3_free(aIter);
   389    374     return rc;
   390    375   }
................................................................................
   413    398     int i;
   414    399     int rc = SQLITE_OK;
   415    400     
   416    401     fts5BufferZero(&pPhrase->poslist);
   417    402   
   418    403     /* If the aStatic[] array is not large enough, allocate a large array
   419    404     ** using sqlite3_malloc(). This approach could be improved upon. */
   420         -  if( pPhrase->nTerm>(int)ArraySize(aStatic) ){
          405  +  if( pPhrase->nTerm>ArraySize(aStatic) ){
   421    406       int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm;
   422    407       aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte);
   423    408       if( !aIter ) return SQLITE_NOMEM;
   424    409     }
   425    410     memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm);
   426    411   
   427    412     /* Initialize a term iterator for each term in the phrase */
   428    413     for(i=0; i<pPhrase->nTerm; i++){
   429    414       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
   430         -    i64 dummy;
   431    415       int n = 0;
   432    416       int bFlag = 0;
   433         -    const u8 *a = 0;
          417  +    u8 *a = 0;
   434    418       if( pTerm->pSynonym ){
          419  +      Fts5Buffer buf = {0, 0, 0};
   435    420         rc = fts5ExprSynonymList(
   436         -          pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
          421  +          pTerm, 0, pColset, pNode->iRowid, &buf, &a, &n
   437    422         );
          423  +      if( rc ){
          424  +        sqlite3_free(a);
          425  +        goto ismatch_out;
          426  +      }
          427  +      if( a==buf.p ) bFlag = 1;
   438    428       }else{
   439         -      rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
          429  +      a = (u8*)pTerm->pIter->pData;
          430  +      n = pTerm->pIter->nData;
   440    431       }
   441         -    if( rc!=SQLITE_OK ) goto ismatch_out;
   442    432       sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
   443    433       aIter[i].bFlag = (u8)bFlag;
   444    434       if( aIter[i].bEof ) goto ismatch_out;
   445    435     }
   446    436   
   447    437     while( 1 ){
   448    438       int bMatch;
................................................................................
   549    539     int rc = *pRc;
   550    540     int bMatch;
   551    541   
   552    542     assert( pNear->nPhrase>1 );
   553    543   
   554    544     /* If the aStatic[] array is not large enough, allocate a large array
   555    545     ** using sqlite3_malloc(). This approach could be improved upon. */
   556         -  if( pNear->nPhrase>(int)ArraySize(aStatic) ){
          546  +  if( pNear->nPhrase>ArraySize(aStatic) ){
   557    547       int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase;
   558    548       a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte);
   559    549     }else{
   560    550       memset(aStatic, 0, sizeof(aStatic));
   561    551     }
   562    552     if( rc!=SQLITE_OK ){
   563    553       *pRc = rc;
................................................................................
   771    761     if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
   772    762       Fts5ExprTerm *pTerm;
   773    763       Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
   774    764       pPhrase->poslist.n = 0;
   775    765       for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
   776    766         Fts5IndexIter *pIter = pTerm->pIter;
   777    767         if( sqlite3Fts5IterEof(pIter)==0 ){
          768  +#if 0
   778    769           int n;
   779    770           i64 iRowid;
   780    771           rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
   781    772           if( rc!=SQLITE_OK ){
   782    773             *pRc = rc;
   783    774             return 0;
   784    775           }else if( iRowid==pNode->iRowid && n>0 ){
   785    776             pPhrase->poslist.n = 1;
   786    777           }
          778  +#endif
          779  +        if( pIter->iRowid==pNode->iRowid && pIter->nData>0 ){
          780  +          pPhrase->poslist.n = 1;
          781  +        }
   787    782         }
   788    783       }
   789    784       return pPhrase->poslist.n;
   790    785     }else{
   791    786       int i;
   792    787   
   793    788       /* Check that each phrase in the nearset matches the current row.
................................................................................
   796    791       for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
   797    792         Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
   798    793         if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
   799    794           int bMatch = 0;
   800    795           rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
   801    796           if( bMatch==0 ) break;
   802    797         }else{
          798  +        Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
          799  +        fts5BufferSet(&rc, &pPhrase->poslist, pIter->nData, pIter->pData);
          800  +#if 0
   803    801           rc = sqlite3Fts5IterPoslistBuffer(
   804    802               pPhrase->aTerm[0].pIter, &pPhrase->poslist
   805    803           );
          804  +#endif
   806    805         }
   807    806       }
   808    807   
   809    808       *pRc = rc;
   810    809       if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
   811    810         return 1;
   812    811       }
................................................................................
   819    818     Fts5ExprNode *pNode             /* The "NEAR" node (FTS5_TERM) */
   820    819   ){
   821    820     /* As this "NEAR" object is actually a single phrase that consists 
   822    821     ** of a single term only, grab pointers into the poslist managed by the
   823    822     ** fts5_index.c iterator object. This is much faster than synthesizing 
   824    823     ** a new poslist the way we have to for more complicated phrase or NEAR
   825    824     ** expressions.  */
   826         -  Fts5ExprNearset *pNear = pNode->pNear;
   827         -  Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
          825  +  Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0];
   828    826     Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter;
   829         -  Fts5Colset *pColset = pNear->pColset;
   830         -  int rc;
   831    827   
   832    828     assert( pNode->eType==FTS5_TERM );
   833         -  assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
          829  +  assert( pNode->pNear->nPhrase==1 && pPhrase->nTerm==1 );
   834    830     assert( pPhrase->aTerm[0].pSynonym==0 );
   835    831   
   836         -  rc = sqlite3Fts5IterPoslist(pIter, pColset, 
   837         -      (const u8**)&pPhrase->poslist.p, (int*)&pPhrase->poslist.n, &pNode->iRowid
   838         -  );
          832  +  pPhrase->poslist.n = pIter->nData;
          833  +  if( pExpr->pConfig->eDetail==FTS5_DETAIL_FULL ){
          834  +    pPhrase->poslist.p = (u8*)pIter->pData;
          835  +  }
          836  +  pNode->iRowid = pIter->iRowid;
   839    837     pNode->bNomatch = (pPhrase->poslist.n==0);
   840         -  return rc;
          838  +  return SQLITE_OK;
   841    839   }
   842    840   
   843    841   /*
   844    842   ** All individual term iterators in pNear are guaranteed to be valid when
   845    843   ** this function is called. This function checks if all term iterators
   846    844   ** point to the same rowid, and if not, advances them until they do.
   847    845   ** If an EOF is reached before this happens, *pbEof is set to true before
................................................................................
  1366   1364       int i;
  1367   1365       for(i=0; i<pPhrase->nTerm; i++){
  1368   1366         Fts5ExprTerm *pSyn;
  1369   1367         Fts5ExprTerm *pNext;
  1370   1368         Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
  1371   1369         sqlite3_free(pTerm->zTerm);
  1372   1370         sqlite3Fts5IterClose(pTerm->pIter);
  1373         -
  1374   1371         for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
  1375   1372           pNext = pSyn->pSynonym;
  1376   1373           sqlite3Fts5IterClose(pSyn->pIter);
         1374  +        fts5BufferFree((Fts5Buffer*)&pSyn[1]);
  1377   1375           sqlite3_free(pSyn);
  1378   1376         }
  1379   1377       }
  1380   1378       if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
  1381   1379       sqlite3_free(pPhrase);
  1382   1380     }
  1383   1381   }
................................................................................
  1457   1455   
  1458   1456     /* If an error has already occurred, this is a no-op */
  1459   1457     if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
  1460   1458   
  1461   1459     assert( pPhrase==0 || pPhrase->nTerm>0 );
  1462   1460     if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
  1463   1461       Fts5ExprTerm *pSyn;
  1464         -    int nByte = sizeof(Fts5ExprTerm) + nToken+1;
         1462  +    int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1;
  1465   1463       pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
  1466   1464       if( pSyn==0 ){
  1467   1465         rc = SQLITE_NOMEM;
  1468   1466       }else{
  1469   1467         memset(pSyn, 0, nByte);
  1470         -      pSyn->zTerm = (char*)&pSyn[1];
         1468  +      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
  1471   1469         memcpy(pSyn->zTerm, pToken, nToken);
  1472   1470         pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
  1473   1471         pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
  1474   1472       }
  1475   1473     }else{
  1476   1474       Fts5ExprTerm *pTerm;
  1477   1475       if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
................................................................................
  2242   2240       { "fts5_isalnum",  fts5ExprIsAlnum },
  2243   2241       { "fts5_fold",     fts5ExprFold },
  2244   2242     };
  2245   2243     int i;
  2246   2244     int rc = SQLITE_OK;
  2247   2245     void *pCtx = (void*)pGlobal;
  2248   2246   
  2249         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aFunc); i++){
         2247  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aFunc); i++){
  2250   2248       struct Fts5ExprFunc *p = &aFunc[i];
  2251   2249       rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0);
  2252   2250     }
  2253   2251   
  2254   2252     /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */
  2255   2253   #ifndef NDEBUG
  2256   2254     (void)sqlite3Fts5ParserTrace;
................................................................................
  2480   2478     int *pnCollist
  2481   2479   ){
  2482   2480     Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
  2483   2481     Fts5ExprNode *pNode = pPhrase->pNode;
  2484   2482     int rc = SQLITE_OK;
  2485   2483   
  2486   2484     assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
         2485  +  assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         2486  +
  2487   2487     if( pNode->bEof==0 
  2488   2488      && pNode->iRowid==pExpr->pRoot->iRowid 
  2489   2489      && pPhrase->poslist.n>0
  2490   2490     ){
  2491   2491       Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
  2492   2492       if( pTerm->pSynonym ){
  2493         -      int bDel = 0;
  2494         -      u8 *a;
         2493  +      Fts5Buffer *pBuf = (Fts5Buffer*)&pTerm->pSynonym[1];
  2495   2494         rc = fts5ExprSynonymList(
  2496         -          pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
         2495  +          pTerm, 1, 0, pNode->iRowid, pBuf, (u8**)ppCollist, pnCollist
  2497   2496         );
  2498         -      if( bDel ){
  2499         -        sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
  2500         -        *ppCollist = pPhrase->poslist.p;
  2501         -        sqlite3_free(a);
  2502         -      }else{
  2503         -        *ppCollist = a;
  2504         -      }
  2505   2497       }else{
  2506         -      sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
         2498  +      *ppCollist = pPhrase->aTerm[0].pIter->pData;
         2499  +      *pnCollist = pPhrase->aTerm[0].pIter->nData;
  2507   2500       }
  2508   2501     }else{
  2509   2502       *ppCollist = 0;
  2510   2503       *pnCollist = 0;
  2511   2504     }
  2512   2505   
  2513   2506     return rc;
  2514   2507   }
  2515   2508   

Changes to ext/fts5/fts5_index.c.

   257    257   #define FTS5_DATA_ZERO_PADDING 8
   258    258   #define FTS5_DATA_PADDING 20
   259    259   
   260    260   typedef struct Fts5Data Fts5Data;
   261    261   typedef struct Fts5DlidxIter Fts5DlidxIter;
   262    262   typedef struct Fts5DlidxLvl Fts5DlidxLvl;
   263    263   typedef struct Fts5DlidxWriter Fts5DlidxWriter;
          264  +typedef struct Fts5Iter Fts5Iter;
   264    265   typedef struct Fts5PageWriter Fts5PageWriter;
   265    266   typedef struct Fts5SegIter Fts5SegIter;
   266    267   typedef struct Fts5DoclistIter Fts5DoclistIter;
   267    268   typedef struct Fts5SegWriter Fts5SegWriter;
   268    269   typedef struct Fts5Structure Fts5Structure;
   269    270   typedef struct Fts5StructureLevel Fts5StructureLevel;
   270    271   typedef struct Fts5StructureSegment Fts5StructureSegment;
................................................................................
   499    500   ** aFirst[1] contains the index in aSeg[] of the iterator that points to
   500    501   ** the smallest key overall. aFirst[0] is unused. 
   501    502   **
   502    503   ** poslist:
   503    504   **   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
   504    505   **   There is no way to tell if this is populated or not.
   505    506   */
   506         -struct Fts5IndexIter {
          507  +struct Fts5Iter {
          508  +  Fts5IndexIter base;             /* Base class containing output vars */
          509  +
   507    510     Fts5Index *pIndex;              /* Index that owns this iterator */
   508    511     Fts5Structure *pStruct;         /* Database structure for this iterator */
   509    512     Fts5Buffer poslist;             /* Buffer containing current poslist */
          513  +  Fts5Colset *pColset;            /* Restrict matches to these columns */
          514  +
          515  +  /* Invoked to set output variables. */
          516  +  void (*xSetOutputs)(Fts5Iter*, Fts5SegIter*);
   510    517   
   511    518     int nSeg;                       /* Size of aSeg[] array */
   512    519     int bRev;                       /* True to iterate in reverse order */
   513    520     u8 bSkipEmpty;                  /* True to skip deleted entries */
   514    521     u8 bEof;                        /* True at EOF */
   515    522     u8 bFiltered;                   /* True if column-filter already applied */
   516    523   
................................................................................
  1748   1755   }
  1749   1756   
  1750   1757   /*
  1751   1758   ** Return true if the iterator passed as the second argument currently
  1752   1759   ** points to a delete marker. A delete marker is an entry with a 0 byte
  1753   1760   ** position-list.
  1754   1761   */
  1755         -static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5IndexIter *pIter){
         1762  +static int fts5MultiIterIsEmpty(Fts5Index *p, Fts5Iter *pIter){
  1756   1763     Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
  1757   1764     return (p->rc==SQLITE_OK && pSeg->pLeaf && pSeg->nPos==0);
  1758   1765   }
  1759   1766   
  1760   1767   /*
  1761   1768   ** Advance iterator pIter to the next entry.
  1762   1769   **
................................................................................
  2402   2409   /*
  2403   2410   ** This function is used as part of the big assert() procedure implemented by
  2404   2411   ** fts5AssertMultiIterSetup(). It ensures that the result currently stored
  2405   2412   ** in *pRes is the correct result of comparing the current positions of the
  2406   2413   ** two iterators.
  2407   2414   */
  2408   2415   static void fts5AssertComparisonResult(
  2409         -  Fts5IndexIter *pIter, 
         2416  +  Fts5Iter *pIter, 
  2410   2417     Fts5SegIter *p1,
  2411   2418     Fts5SegIter *p2,
  2412   2419     Fts5CResult *pRes
  2413   2420   ){
  2414   2421     int i1 = p1 - pIter->aSeg;
  2415   2422     int i2 = p2 - pIter->aSeg;
  2416   2423   
................................................................................
  2443   2450   
  2444   2451   /*
  2445   2452   ** This function is a no-op unless SQLITE_DEBUG is defined when this module
  2446   2453   ** is compiled. In that case, this function is essentially an assert() 
  2447   2454   ** statement used to verify that the contents of the pIter->aFirst[] array
  2448   2455   ** are correct.
  2449   2456   */
  2450         -static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5IndexIter *pIter){
         2457  +static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){
  2451   2458     if( p->rc==SQLITE_OK ){
  2452   2459       Fts5SegIter *pFirst = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2453   2460       int i;
  2454   2461   
  2455   2462       assert( (pFirst->pLeaf==0)==pIter->bEof );
  2456   2463   
  2457   2464       /* Check that pIter->iSwitchRowid is set correctly. */
................................................................................
  2488   2495   ** Do the comparison necessary to populate pIter->aFirst[iOut].
  2489   2496   **
  2490   2497   ** If the returned value is non-zero, then it is the index of an entry
  2491   2498   ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing
  2492   2499   ** to a key that is a duplicate of another, higher priority, 
  2493   2500   ** segment-iterator in the pSeg->aSeg[] array.
  2494   2501   */
  2495         -static int fts5MultiIterDoCompare(Fts5IndexIter *pIter, int iOut){
         2502  +static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
  2496   2503     int i1;                         /* Index of left-hand Fts5SegIter */
  2497   2504     int i2;                         /* Index of right-hand Fts5SegIter */
  2498   2505     int iRes;
  2499   2506     Fts5SegIter *p1;                /* Left-hand Fts5SegIter */
  2500   2507     Fts5SegIter *p2;                /* Right-hand Fts5SegIter */
  2501   2508     Fts5CResult *pRes = &pIter->aFirst[iOut];
  2502   2509   
................................................................................
  2634   2641     }while( p->rc==SQLITE_OK );
  2635   2642   }
  2636   2643   
  2637   2644   
  2638   2645   /*
  2639   2646   ** Free the iterator object passed as the second argument.
  2640   2647   */
  2641         -static void fts5MultiIterFree(Fts5Index *p, Fts5IndexIter *pIter){
         2648  +static void fts5MultiIterFree(Fts5Index *p, Fts5Iter *pIter){
  2642   2649     if( pIter ){
  2643   2650       int i;
  2644   2651       for(i=0; i<pIter->nSeg; i++){
  2645   2652         fts5SegIterClear(&pIter->aSeg[i]);
  2646   2653       }
  2647   2654       fts5StructureRelease(pIter->pStruct);
  2648   2655       fts5BufferFree(&pIter->poslist);
  2649   2656       sqlite3_free(pIter);
  2650   2657     }
  2651   2658   }
  2652   2659   
  2653   2660   static void fts5MultiIterAdvanced(
  2654   2661     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2655         -  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
         2662  +  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
  2656   2663     int iChanged,                   /* Index of sub-iterator just advanced */
  2657   2664     int iMinset                     /* Minimum entry in aFirst[] to set */
  2658   2665   ){
  2659   2666     int i;
  2660   2667     for(i=(pIter->nSeg+iChanged)/2; i>=iMinset && p->rc==SQLITE_OK; i=i/2){
  2661   2668       int iEq;
  2662   2669       if( (iEq = fts5MultiIterDoCompare(pIter, i)) ){
................................................................................
  2676   2683   **
  2677   2684   ** If non-zero is returned, the caller should call fts5MultiIterAdvanced()
  2678   2685   ** on the iterator instead. That function does the same as this one, except
  2679   2686   ** that it deals with more complicated cases as well.
  2680   2687   */ 
  2681   2688   static int fts5MultiIterAdvanceRowid(
  2682   2689     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2683         -  Fts5IndexIter *pIter,           /* Iterator to update aFirst[] array for */
  2684         -  int iChanged                    /* Index of sub-iterator just advanced */
         2690  +  Fts5Iter *pIter,                /* Iterator to update aFirst[] array for */
         2691  +  int iChanged,                   /* Index of sub-iterator just advanced */
         2692  +  Fts5SegIter **ppFirst
  2685   2693   ){
  2686   2694     Fts5SegIter *pNew = &pIter->aSeg[iChanged];
  2687   2695   
  2688   2696     if( pNew->iRowid==pIter->iSwitchRowid
  2689   2697      || (pNew->iRowid<pIter->iSwitchRowid)==pIter->bRev
  2690   2698     ){
  2691   2699       int i;
................................................................................
  2710   2718         pRes->iFirst = (u16)(pNew - pIter->aSeg);
  2711   2719         if( i==1 ) break;
  2712   2720   
  2713   2721         pOther = &pIter->aSeg[ pIter->aFirst[i ^ 0x0001].iFirst ];
  2714   2722       }
  2715   2723     }
  2716   2724   
         2725  +  *ppFirst = pNew;
  2717   2726     return 0;
  2718   2727   }
  2719   2728   
  2720   2729   /*
  2721   2730   ** Set the pIter->bEof variable based on the state of the sub-iterators.
  2722   2731   */
  2723         -static void fts5MultiIterSetEof(Fts5IndexIter *pIter){
         2732  +static void fts5MultiIterSetEof(Fts5Iter *pIter){
  2724   2733     Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  2725   2734     pIter->bEof = pSeg->pLeaf==0;
  2726   2735     pIter->iSwitchRowid = pSeg->iRowid;
  2727   2736   }
  2728   2737   
  2729   2738   /*
  2730   2739   ** Move the iterator to the next entry. 
................................................................................
  2731   2740   **
  2732   2741   ** If an error occurs, an error code is left in Fts5Index.rc. It is not 
  2733   2742   ** considered an error if the iterator reaches EOF, or if it is already at 
  2734   2743   ** EOF when this function is called.
  2735   2744   */
  2736   2745   static void fts5MultiIterNext(
  2737   2746     Fts5Index *p, 
  2738         -  Fts5IndexIter *pIter,
         2747  +  Fts5Iter *pIter,
  2739   2748     int bFrom,                      /* True if argument iFrom is valid */
  2740   2749     i64 iFrom                       /* Advance at least as far as this */
  2741   2750   ){
  2742         -  if( p->rc==SQLITE_OK ){
  2743         -    int bUseFrom = bFrom;
  2744         -    do {
  2745         -      int iFirst = pIter->aFirst[1].iFirst;
  2746         -      int bNewTerm = 0;
  2747         -      Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2748         -      assert( p->rc==SQLITE_OK );
  2749         -      if( bUseFrom && pSeg->pDlidx ){
  2750         -        fts5SegIterNextFrom(p, pSeg, iFrom);
  2751         -      }else{
  2752         -        pSeg->xNext(p, pSeg, &bNewTerm);
  2753         -      }
  2754         -
  2755         -      if( pSeg->pLeaf==0 || bNewTerm 
  2756         -       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
  2757         -      ){
  2758         -        fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2759         -        fts5MultiIterSetEof(pIter);
  2760         -      }
  2761         -      fts5AssertMultiIterSetup(p, pIter);
  2762         -
  2763         -      bUseFrom = 0;
  2764         -    }while( pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter) );
         2751  +  int bUseFrom = bFrom;
         2752  +  while( p->rc==SQLITE_OK ){
         2753  +    int iFirst = pIter->aFirst[1].iFirst;
         2754  +    int bNewTerm = 0;
         2755  +    Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
         2756  +    assert( p->rc==SQLITE_OK );
         2757  +    if( bUseFrom && pSeg->pDlidx ){
         2758  +      fts5SegIterNextFrom(p, pSeg, iFrom);
         2759  +    }else{
         2760  +      pSeg->xNext(p, pSeg, &bNewTerm);
         2761  +    }
         2762  +
         2763  +    if( pSeg->pLeaf==0 || bNewTerm 
         2764  +     || fts5MultiIterAdvanceRowid(p, pIter, iFirst, &pSeg)
         2765  +    ){
         2766  +      fts5MultiIterAdvanced(p, pIter, iFirst, 1);
         2767  +      fts5MultiIterSetEof(pIter);
         2768  +      pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
         2769  +      if( pSeg->pLeaf==0 ) return;
         2770  +    }
         2771  +
         2772  +    fts5AssertMultiIterSetup(p, pIter);
         2773  +    assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf );
         2774  +    if( pIter->bSkipEmpty==0 || pSeg->nPos ){
         2775  +      pIter->xSetOutputs(pIter, pSeg);
         2776  +      return;
         2777  +    }
         2778  +    bUseFrom = 0;
  2765   2779     }
  2766   2780   }
  2767   2781   
  2768   2782   static void fts5MultiIterNext2(
  2769   2783     Fts5Index *p, 
  2770         -  Fts5IndexIter *pIter,
         2784  +  Fts5Iter *pIter,
  2771   2785     int *pbNewTerm                  /* OUT: True if *might* be new term */
  2772   2786   ){
  2773   2787     assert( pIter->bSkipEmpty );
  2774   2788     if( p->rc==SQLITE_OK ){
  2775   2789       do {
  2776   2790         int iFirst = pIter->aFirst[1].iFirst;
  2777   2791         Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
  2778   2792         int bNewTerm = 0;
  2779   2793   
  2780   2794         assert( p->rc==SQLITE_OK );
  2781   2795         pSeg->xNext(p, pSeg, &bNewTerm);
  2782   2796         if( pSeg->pLeaf==0 || bNewTerm 
  2783         -       || fts5MultiIterAdvanceRowid(p, pIter, iFirst)
         2797  +       || fts5MultiIterAdvanceRowid(p, pIter, iFirst, &pSeg)
  2784   2798         ){
  2785   2799           fts5MultiIterAdvanced(p, pIter, iFirst, 1);
  2786   2800           fts5MultiIterSetEof(pIter);
  2787   2801           *pbNewTerm = 1;
  2788   2802         }else{
  2789   2803           *pbNewTerm = 0;
  2790   2804         }
  2791   2805         fts5AssertMultiIterSetup(p, pIter);
  2792   2806   
  2793   2807       }while( fts5MultiIterIsEmpty(p, pIter) );
  2794   2808     }
  2795   2809   }
  2796   2810   
         2811  +static void fts5IterSetOutputs_Noop(Fts5Iter *pIter, Fts5SegIter *pSeg){
         2812  +}
  2797   2813   
  2798         -static Fts5IndexIter *fts5MultiIterAlloc(
         2814  +static Fts5Iter *fts5MultiIterAlloc(
  2799   2815     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2800   2816     int nSeg
  2801   2817   ){
  2802         -  Fts5IndexIter *pNew;
         2818  +  Fts5Iter *pNew;
  2803   2819     int nSlot;                      /* Power of two >= nSeg */
  2804   2820   
  2805   2821     for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
  2806   2822     pNew = fts5IdxMalloc(p, 
  2807         -      sizeof(Fts5IndexIter) +             /* pNew */
         2823  +      sizeof(Fts5Iter) +                  /* pNew */
  2808   2824         sizeof(Fts5SegIter) * (nSlot-1) +   /* pNew->aSeg[] */
  2809   2825         sizeof(Fts5CResult) * nSlot         /* pNew->aFirst[] */
  2810   2826     );
  2811   2827     if( pNew ){
  2812   2828       pNew->nSeg = nSlot;
  2813   2829       pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
  2814   2830       pNew->pIndex = p;
         2831  +    pNew->xSetOutputs = fts5IterSetOutputs_Noop;
  2815   2832     }
  2816   2833     return pNew;
  2817   2834   }
  2818   2835   
  2819   2836   /*
  2820         -** Allocate a new Fts5IndexIter object.
         2837  +** Allocate a new Fts5Iter object.
  2821   2838   **
  2822   2839   ** The new object will be used to iterate through data in structure pStruct.
  2823   2840   ** If iLevel is -ve, then all data in all segments is merged. Or, if iLevel
  2824   2841   ** is zero or greater, data from the first nSegment segments on level iLevel
  2825   2842   ** is merged.
  2826   2843   **
  2827   2844   ** The iterator initially points to the first term/rowid entry in the 
................................................................................
  2831   2848     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2832   2849     Fts5Structure *pStruct,         /* Structure of specific index */
  2833   2850     int bSkipEmpty,                 /* True to ignore delete-keys */
  2834   2851     int flags,                      /* FTS5INDEX_QUERY_XXX flags */
  2835   2852     const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
  2836   2853     int iLevel,                     /* Level to iterate (-1 for all) */
  2837   2854     int nSegment,                   /* Number of segments to merge (iLevel>=0) */
  2838         -  Fts5IndexIter **ppOut           /* New object */
         2855  +  Fts5Iter **ppOut                /* New object */
  2839   2856   ){
  2840   2857     int nSeg = 0;                   /* Number of segment-iters in use */
  2841   2858     int iIter = 0;                  /* */
  2842   2859     int iSeg;                       /* Used to iterate through segments */
  2843   2860     Fts5Buffer buf = {0,0,0};       /* Buffer used by fts5SegIterSeekInit() */
  2844   2861     Fts5StructureLevel *pLvl;
  2845         -  Fts5IndexIter *pNew;
         2862  +  Fts5Iter *pNew;
  2846   2863   
  2847   2864     assert( (pTerm==0 && nTerm==0) || iLevel<0 );
  2848   2865   
  2849   2866     /* Allocate space for the new multi-seg-iterator. */
  2850   2867     if( p->rc==SQLITE_OK ){
  2851   2868       if( iLevel<0 ){
  2852   2869         assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
................................................................................
  2913   2930       fts5MultiIterFree(p, pNew);
  2914   2931       *ppOut = 0;
  2915   2932     }
  2916   2933     fts5BufferFree(&buf);
  2917   2934   }
  2918   2935   
  2919   2936   /*
  2920         -** Create an Fts5IndexIter that iterates through the doclist provided
         2937  +** Create an Fts5Iter that iterates through the doclist provided
  2921   2938   ** as the second argument.
  2922   2939   */
  2923   2940   static void fts5MultiIterNew2(
  2924   2941     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2925   2942     Fts5Data *pData,                /* Doclist to iterate through */
  2926   2943     int bDesc,                      /* True for descending rowid order */
  2927         -  Fts5IndexIter **ppOut           /* New object */
         2944  +  Fts5Iter **ppOut                /* New object */
  2928   2945   ){
  2929         -  Fts5IndexIter *pNew;
         2946  +  Fts5Iter *pNew;
  2930   2947     pNew = fts5MultiIterAlloc(p, 2);
  2931   2948     if( pNew ){
  2932   2949       Fts5SegIter *pIter = &pNew->aSeg[1];
  2933   2950   
  2934   2951       pNew->bFiltered = 1;
  2935   2952       pIter->flags = FTS5_SEGITER_ONETERM;
  2936   2953       if( pData->szLeaf>0 ){
................................................................................
  2957   2974     fts5DataRelease(pData);
  2958   2975   }
  2959   2976   
  2960   2977   /*
  2961   2978   ** Return true if the iterator is at EOF or if an error has occurred. 
  2962   2979   ** False otherwise.
  2963   2980   */
  2964         -static int fts5MultiIterEof(Fts5Index *p, Fts5IndexIter *pIter){
         2981  +static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){
  2965   2982     assert( p->rc 
  2966   2983         || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->bEof 
  2967   2984     );
  2968   2985     return (p->rc || pIter->bEof);
  2969   2986   }
  2970   2987   
  2971   2988   /*
  2972   2989   ** Return the rowid of the entry that the iterator currently points
  2973   2990   ** to. If the iterator points to EOF when this function is called the
  2974   2991   ** results are undefined.
  2975   2992   */
  2976         -static i64 fts5MultiIterRowid(Fts5IndexIter *pIter){
         2993  +static i64 fts5MultiIterRowid(Fts5Iter *pIter){
  2977   2994     assert( pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf );
  2978   2995     return pIter->aSeg[ pIter->aFirst[1].iFirst ].iRowid;
  2979   2996   }
  2980   2997   
  2981   2998   /*
  2982   2999   ** Move the iterator to the next entry at or following iMatch.
  2983   3000   */
  2984   3001   static void fts5MultiIterNextFrom(
  2985   3002     Fts5Index *p, 
  2986         -  Fts5IndexIter *pIter, 
         3003  +  Fts5Iter *pIter, 
  2987   3004     i64 iMatch
  2988   3005   ){
  2989   3006     while( 1 ){
  2990   3007       i64 iRowid;
  2991   3008       fts5MultiIterNext(p, pIter, 1, iMatch);
  2992   3009       if( fts5MultiIterEof(p, pIter) ) break;
  2993   3010       iRowid = fts5MultiIterRowid(pIter);
................................................................................
  2996   3013     }
  2997   3014   }
  2998   3015   
  2999   3016   /*
  3000   3017   ** Return a pointer to a buffer containing the term associated with the 
  3001   3018   ** entry that the iterator currently points to.
  3002   3019   */
  3003         -static const u8 *fts5MultiIterTerm(Fts5IndexIter *pIter, int *pn){
         3020  +static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){
  3004   3021     Fts5SegIter *p = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  3005   3022     *pn = p->term.n;
  3006   3023     return p->term.p;
  3007   3024   }
  3008   3025   
  3009   3026   static void fts5ChunkIterate(
  3010   3027     Fts5Index *p,                   /* Index object */
................................................................................
  3578   3595   }
  3579   3596   
  3580   3597   /*
  3581   3598   ** Iterator pIter was used to iterate through the input segments of on an
  3582   3599   ** incremental merge operation. This function is called if the incremental
  3583   3600   ** merge step has finished but the input has not been completely exhausted.
  3584   3601   */
  3585         -static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
         3602  +static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
  3586   3603     int i;
  3587   3604     Fts5Buffer buf;
  3588   3605     memset(&buf, 0, sizeof(Fts5Buffer));
  3589   3606     for(i=0; i<pIter->nSeg; i++){
  3590   3607       Fts5SegIter *pSeg = &pIter->aSeg[i];
  3591   3608       if( pSeg->pSeg==0 ){
  3592   3609         /* no-op */
................................................................................
  3656   3673     Fts5Structure **ppStruct,       /* IN/OUT: Stucture of index */
  3657   3674     int iLvl,                       /* Level to read input from */
  3658   3675     int *pnRem                      /* Write up to this many output leaves */
  3659   3676   ){
  3660   3677     Fts5Structure *pStruct = *ppStruct;
  3661   3678     Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
  3662   3679     Fts5StructureLevel *pLvlOut;
  3663         -  Fts5IndexIter *pIter = 0;       /* Iterator to read input data */
         3680  +  Fts5Iter *pIter = 0;       /* Iterator to read input data */
  3664   3681     int nRem = pnRem ? *pnRem : 0;  /* Output leaf pages left to write */
  3665   3682     int nInput;                     /* Number of input segments */
  3666   3683     Fts5SegWriter writer;           /* Writer object */
  3667   3684     Fts5StructureSegment *pSeg;     /* Output segment */
  3668   3685     Fts5Buffer term;
  3669   3686     int bOldest;                    /* True if the output segment is the oldest */
  3670   3687     int eDetail = p->pConfig->eDetail;
................................................................................
  4338   4355     }
  4339   4356     return p - (*pa);
  4340   4357   }
  4341   4358   
  4342   4359   static int fts5AppendRowid(
  4343   4360     Fts5Index *p,
  4344   4361     i64 iDelta,
  4345         -  Fts5IndexIter *pMulti,
         4362  +  Fts5Iter *pMulti,
  4346   4363     Fts5Colset *pColset,
  4347   4364     Fts5Buffer *pBuf
  4348   4365   ){
  4349   4366     fts5BufferAppendVarint(&p->rc, pBuf, iDelta);
  4350   4367     return 0;
  4351   4368   }
  4352   4369   
................................................................................
  4363   4380   ** even iDelta).
  4364   4381   **
  4365   4382   ** If an error occurs, an error code is left in p->rc. 
  4366   4383   */
  4367   4384   static int fts5AppendPoslist(
  4368   4385     Fts5Index *p,
  4369   4386     i64 iDelta,
  4370         -  Fts5IndexIter *pMulti,
         4387  +  Fts5Iter *pMulti,
  4371   4388     Fts5Colset *pColset,
  4372   4389     Fts5Buffer *pBuf
  4373   4390   ){
  4374   4391     if( p->rc==SQLITE_OK ){
  4375   4392       Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
  4376   4393       assert( fts5MultiIterEof(p, pMulti)==0 );
  4377   4394       assert( pSeg->nPos>0 );
................................................................................
  4641   4658   
  4642   4659   static void fts5SetupPrefixIter(
  4643   4660     Fts5Index *p,                   /* Index to read from */
  4644   4661     int bDesc,                      /* True for "ORDER BY rowid DESC" */
  4645   4662     const u8 *pToken,               /* Buffer containing prefix to match */
  4646   4663     int nToken,                     /* Size of buffer pToken in bytes */
  4647   4664     Fts5Colset *pColset,            /* Restrict matches to these columns */
  4648         -  Fts5IndexIter **ppIter          /* OUT: New iterator */
         4665  +  Fts5Iter **ppIter          /* OUT: New iterator */
  4649   4666   ){
  4650   4667     Fts5Structure *pStruct;
  4651   4668     Fts5Buffer *aBuf;
  4652   4669     const int nBuf = 32;
  4653   4670   
  4654   4671     void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*);
  4655         -  int (*xAppend)(Fts5Index*, i64, Fts5IndexIter*, Fts5Colset*, Fts5Buffer*);
         4672  +  int (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Colset*, Fts5Buffer*);
  4656   4673     if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
  4657   4674       xMerge = fts5MergeRowidLists;
  4658   4675       xAppend = fts5AppendRowid;
  4659   4676     }else{
  4660   4677       xMerge = fts5MergePrefixLists;
  4661   4678       xAppend = fts5AppendPoslist;
  4662   4679     }
................................................................................
  4664   4681     aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  4665   4682     pStruct = fts5StructureRead(p);
  4666   4683   
  4667   4684     if( aBuf && pStruct ){
  4668   4685       const int flags = FTS5INDEX_QUERY_SCAN;
  4669   4686       int i;
  4670   4687       i64 iLastRowid = 0;
  4671         -    Fts5IndexIter *p1 = 0;     /* Iterator used to gather data from index */
         4688  +    Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
  4672   4689       Fts5Data *pData;
  4673   4690       Fts5Buffer doclist;
  4674   4691       int bNewTerm = 1;
  4675   4692   
  4676   4693       memset(&doclist, 0, sizeof(doclist));
  4677   4694       for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1);
  4678   4695           fts5MultiIterEof(p, p1)==0;
................................................................................
  4928   4945         );
  4929   4946       }
  4930   4947     }
  4931   4948   
  4932   4949     return rc;
  4933   4950   }
  4934   4951   
         4952  +
         4953  +static int fts5IndexExtractColset (
         4954  +  Fts5Colset *pColset,            /* Colset to filter on */
         4955  +  const u8 *pPos, int nPos,       /* Position list */
         4956  +  Fts5Buffer *pBuf                /* Output buffer */
         4957  +){
         4958  +  int rc = SQLITE_OK;
         4959  +  int i;
         4960  +
         4961  +  fts5BufferZero(pBuf);
         4962  +  for(i=0; i<pColset->nCol; i++){
         4963  +    const u8 *pSub = pPos;
         4964  +    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
         4965  +    if( nSub ){
         4966  +      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
         4967  +    }
         4968  +  }
         4969  +  return rc;
         4970  +}
         4971  +
         4972  +/*
         4973  +** xSetOutputs callback used by detail=none tables.
         4974  +*/
         4975  +static void fts5IterSetOutputs_None(Fts5Iter *pIter, Fts5SegIter *pSeg){
         4976  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_NONE );
         4977  +  pIter->base.iRowid = pSeg->iRowid;
         4978  +  pIter->base.nData = pSeg->nPos;
         4979  +}
         4980  +
         4981  +/*
         4982  +** xSetOutputs callback used by detail=full and detail=col tables when no
         4983  +** column filters are specified.
         4984  +*/
         4985  +static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){
         4986  +  pIter->base.iRowid = pSeg->iRowid;
         4987  +  pIter->base.nData = pSeg->nPos;
         4988  +
         4989  +  assert( pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_NONE );
         4990  +  assert( pIter->pColset==0 || pIter->bFiltered );
         4991  +
         4992  +  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         4993  +    /* All data is stored on the current page. Populate the output 
         4994  +    ** variables to point into the body of the page object. */
         4995  +    pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset];
         4996  +  }else{
         4997  +    /* The data is distributed over two or more pages. Copy it into the
         4998  +    ** Fts5Iter.poslist buffer and then set the output pointer to point
         4999  +    ** to this buffer.  */
         5000  +    fts5BufferZero(&pIter->poslist);
         5001  +    fts5SegiterPoslist(pIter->pIndex, pSeg, 0, &pIter->poslist);
         5002  +    pIter->base.pData = pIter->poslist.p;
         5003  +  }
         5004  +}
         5005  +
         5006  +/*
         5007  +** xSetOutputs callback used by detail=col when there is a column filter
         5008  +** and there are 100 or more columns. Also called as a fallback from
         5009  +** fts5IterSetOutputs_Col100 if the column-list spans more than one page.
         5010  +*/
         5011  +static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){
         5012  +  fts5BufferZero(&pIter->poslist);
         5013  +  fts5SegiterPoslist(pIter->pIndex, pSeg, pIter->pColset, &pIter->poslist);
         5014  +  pIter->base.iRowid = pSeg->iRowid;
         5015  +  pIter->base.pData = pIter->poslist.p;
         5016  +  pIter->base.nData = pIter->poslist.n;
         5017  +}
         5018  +
         5019  +/*
         5020  +** xSetOutputs callback used when: 
         5021  +**
         5022  +**   * detail=col,
         5023  +**   * there is a column filter, and
         5024  +**   * the table contains 100 or fewer columns. 
         5025  +**
         5026  +** The last point is to ensure all column numbers are stored as 
         5027  +** single-byte varints.
         5028  +*/
         5029  +static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){
         5030  +
         5031  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         5032  +  assert( pIter->pColset );
         5033  +
         5034  +  if( pSeg->iLeafOffset+pSeg->nPos>pSeg->pLeaf->szLeaf ){
         5035  +    fts5IterSetOutputs_Col(pIter, pSeg);
         5036  +  }else{
         5037  +    u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset];
         5038  +    u8 *pEnd = (u8*)&a[pSeg->nPos]; 
         5039  +    int iPrev = 0;
         5040  +    int *aiCol = pIter->pColset->aiCol;
         5041  +    int *aiColEnd = &aiCol[pIter->pColset->nCol];
         5042  +
         5043  +    u8 *aOut = pIter->poslist.p;
         5044  +    int iPrevOut = 0;
         5045  +
         5046  +    pIter->base.iRowid = pSeg->iRowid;
         5047  +
         5048  +    while( a<pEnd ){
         5049  +      iPrev += (int)a++[0] - 2;
         5050  +      while( *aiCol<iPrev ){
         5051  +        aiCol++;
         5052  +        if( aiCol==aiColEnd ) goto setoutputs_col_out;
         5053  +      }
         5054  +      if( *aiCol==iPrev ){
         5055  +        *aOut++ = (iPrev - iPrevOut) + 2;
         5056  +        iPrevOut = iPrev;
         5057  +      }
         5058  +    }
         5059  +
         5060  +setoutputs_col_out:
         5061  +    pIter->base.pData = pIter->poslist.p;
         5062  +    pIter->base.nData = aOut - pIter->poslist.p;
         5063  +  }
         5064  +}
         5065  +
         5066  +/*
         5067  +** xSetOutputs callback used by detail=full when there is a column filter.
         5068  +*/
         5069  +static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
         5070  +  Fts5Colset *pColset = pIter->pColset;
         5071  +  pIter->base.iRowid = pSeg->iRowid;
         5072  +
         5073  +  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_FULL );
         5074  +  assert( pColset );
         5075  +
         5076  +  if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){
         5077  +    /* All data is stored on the current page. Populate the output 
         5078  +    ** variables to point into the body of the page object. */
         5079  +    const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset];
         5080  +    if( pColset->nCol==1 ){
         5081  +      pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
         5082  +      pIter->base.pData = a;
         5083  +    }else{
         5084  +      fts5BufferZero(&pIter->poslist);
         5085  +      fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
         5086  +      pIter->base.pData = pIter->poslist.p;
         5087  +      pIter->base.nData = pIter->poslist.n;
         5088  +    }
         5089  +  }else{
         5090  +    /* The data is distributed over two or more pages. Copy it into the
         5091  +    ** Fts5Iter.poslist buffer and then set the output pointer to point
         5092  +    ** to this buffer.  */
         5093  +    fts5BufferZero(&pIter->poslist);
         5094  +    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
         5095  +    pIter->base.pData = pIter->poslist.p;
         5096  +    pIter->base.nData = pIter->poslist.n;
         5097  +  }
         5098  +}
         5099  +
         5100  +static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){
         5101  +  Fts5Config *pConfig = pIter->pIndex->pConfig;
         5102  +  if( pConfig->eDetail==FTS5_DETAIL_NONE ){
         5103  +    pIter->xSetOutputs = fts5IterSetOutputs_None;
         5104  +  }
         5105  +
         5106  +  else if( pIter->pColset==0 || pIter->bFiltered ){
         5107  +    pIter->xSetOutputs = fts5IterSetOutputs_Nocolset;
         5108  +  }
         5109  +
         5110  +  else if( pConfig->eDetail==FTS5_DETAIL_FULL ){
         5111  +    pIter->xSetOutputs = fts5IterSetOutputs_Full;
         5112  +  }
         5113  +
         5114  +  else{
         5115  +    assert( pConfig->eDetail==FTS5_DETAIL_COLUMNS );
         5116  +    if( pConfig->nCol<=100 ){
         5117  +      pIter->xSetOutputs = fts5IterSetOutputs_Col100;
         5118  +      sqlite3Fts5BufferSize(pRc, &pIter->poslist, pConfig->nCol);
         5119  +    }else{
         5120  +      pIter->xSetOutputs = fts5IterSetOutputs_Col;
         5121  +    }
         5122  +  }
         5123  +}
         5124  +
  4935   5125   /*
  4936   5126   ** Open a new iterator to iterate though all rowid that match the 
  4937   5127   ** specified token or token prefix.
  4938   5128   */
  4939   5129   int sqlite3Fts5IndexQuery(
  4940   5130     Fts5Index *p,                   /* FTS index to query */
  4941   5131     const char *pToken, int nToken, /* Token (or prefix) to query for */
  4942   5132     int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
  4943   5133     Fts5Colset *pColset,            /* Match these columns only */
  4944   5134     Fts5IndexIter **ppIter          /* OUT: New iterator object */
  4945   5135   ){
  4946   5136     Fts5Config *pConfig = p->pConfig;
  4947         -  Fts5IndexIter *pRet = 0;
  4948         -  int iIdx = 0;
         5137  +  Fts5Iter *pRet = 0;
  4949   5138     Fts5Buffer buf = {0, 0, 0};
  4950   5139   
  4951   5140     /* If the QUERY_SCAN flag is set, all other flags must be clear. */
  4952   5141     assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || flags==FTS5INDEX_QUERY_SCAN );
  4953   5142   
  4954   5143     if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){
         5144  +    int iIdx = 0;                 /* Index to search */
  4955   5145       memcpy(&buf.p[1], pToken, nToken);
  4956   5146   
  4957         -#ifdef SQLITE_DEBUG
  4958         -    /* If the QUERY_TEST_NOIDX flag was specified, then this must be a
         5147  +    /* Figure out which index to search and set iIdx accordingly. If this
         5148  +    ** is a prefix query for which there is no prefix index, set iIdx to
         5149  +    ** greater than pConfig->nPrefix to indicate that the query will be
         5150  +    ** satisfied by scanning multiple terms in the main index.
         5151  +    **
         5152  +    ** If the QUERY_TEST_NOIDX flag was specified, then this must be a
  4959   5153       ** prefix-query. Instead of using a prefix-index (if one exists), 
  4960   5154       ** evaluate the prefix query using the main FTS index. This is used
  4961   5155       ** for internal sanity checking by the integrity-check in debug 
  4962   5156       ** mode only.  */
         5157  +#ifdef SQLITE_DEBUG
  4963   5158       if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){
  4964   5159         assert( flags & FTS5INDEX_QUERY_PREFIX );
  4965   5160         iIdx = 1+pConfig->nPrefix;
  4966   5161       }else
  4967   5162   #endif
  4968   5163       if( flags & FTS5INDEX_QUERY_PREFIX ){
  4969   5164         int nChar = fts5IndexCharlen(pToken, nToken);
  4970   5165         for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
  4971   5166           if( pConfig->aPrefix[iIdx-1]==nChar ) break;
  4972   5167         }
  4973   5168       }
  4974   5169   
  4975   5170       if( iIdx<=pConfig->nPrefix ){
         5171  +      /* Straight index lookup */
  4976   5172         Fts5Structure *pStruct = fts5StructureRead(p);
  4977   5173         buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx);
  4978   5174         if( pStruct ){
  4979   5175           fts5MultiIterNew(p, pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet);
  4980   5176           fts5StructureRelease(pStruct);
  4981   5177         }
  4982   5178       }else{
         5179  +      /* Scan multiple terms in the main index */
  4983   5180         int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
  4984   5181         buf.p[0] = FTS5_MAIN_PREFIX;
  4985   5182         fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet);
  4986   5183       }
  4987   5184   
         5185  +    if( p->rc==SQLITE_OK ){
         5186  +      Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst];
         5187  +      pRet->pColset = pColset;
         5188  +      fts5IterSetOutputCb(&p->rc, pRet);
         5189  +      if( p->rc==SQLITE_OK && pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg);
         5190  +    }
  4988   5191       if( p->rc ){
  4989         -      sqlite3Fts5IterClose(pRet);
         5192  +      sqlite3Fts5IterClose(&pRet->base);
  4990   5193         pRet = 0;
  4991   5194         fts5CloseReader(p);
  4992   5195       }
  4993         -    *ppIter = pRet;
         5196  +
         5197  +    *ppIter = &pRet->base;
  4994   5198       sqlite3Fts5BufferFree(&buf);
  4995   5199     }
  4996   5200     return fts5IndexReturn(p);
  4997   5201   }
  4998   5202   
  4999   5203   /*
  5000   5204   ** Return true if the iterator passed as the only argument is at EOF.
  5001   5205   */
  5002   5206   int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
  5003         -  assert( pIter->pIndex->rc==SQLITE_OK );
  5004         -  return pIter->bEof;
         5207  +  assert( ((Fts5Iter*)pIter)->pIndex->rc==SQLITE_OK );
         5208  +  return ((Fts5Iter*)pIter)->bEof;
  5005   5209   }
  5006   5210   
  5007   5211   /*
  5008   5212   ** Move to the next matching rowid. 
  5009   5213   */
  5010         -int sqlite3Fts5IterNext(Fts5IndexIter *pIter){
         5214  +int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
         5215  +  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5011   5216     assert( pIter->pIndex->rc==SQLITE_OK );
  5012   5217     fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
  5013   5218     return fts5IndexReturn(pIter->pIndex);
  5014   5219   }
  5015   5220   
  5016   5221   /*
  5017   5222   ** Move to the next matching term/rowid. Used by the fts5vocab module.
  5018   5223   */
  5019         -int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){
         5224  +int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
         5225  +  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5020   5226     Fts5Index *p = pIter->pIndex;
  5021   5227   
  5022   5228     assert( pIter->pIndex->rc==SQLITE_OK );
  5023   5229   
  5024   5230     fts5MultiIterNext(p, pIter, 0, 0);
  5025   5231     if( p->rc==SQLITE_OK ){
  5026   5232       Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
................................................................................
  5035   5241   }
  5036   5242   
  5037   5243   /*
  5038   5244   ** Move to the next matching rowid that occurs at or after iMatch. The
  5039   5245   ** definition of "at or after" depends on whether this iterator iterates
  5040   5246   ** in ascending or descending rowid order.
  5041   5247   */
  5042         -int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIter, i64 iMatch){
         5248  +int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
         5249  +  Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5043   5250     fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
  5044   5251     return fts5IndexReturn(pIter->pIndex);
  5045   5252   }
  5046   5253   
  5047   5254   /*
  5048   5255   ** Return the current rowid.
  5049   5256   */
  5050         -i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){
  5051         -  return fts5MultiIterRowid(pIter);
         5257  +i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIndexIter){
         5258  +  return fts5MultiIterRowid((Fts5Iter*)pIndexIter);
  5052   5259   }
  5053   5260   
  5054   5261   /*
  5055   5262   ** Return the current term.
  5056   5263   */
  5057         -const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){
         5264  +const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
  5058   5265     int n;
  5059         -  const char *z = (const char*)fts5MultiIterTerm(pIter, &n);
         5266  +  const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n);
  5060   5267     *pn = n-1;
  5061   5268     return &z[1];
  5062   5269   }
  5063   5270   
  5064         -
  5065         -static int fts5IndexExtractColset (
  5066         -  Fts5Colset *pColset,            /* Colset to filter on */
  5067         -  const u8 *pPos, int nPos,       /* Position list */
  5068         -  Fts5Buffer *pBuf                /* Output buffer */
  5069         -){
  5070         -  int rc = SQLITE_OK;
  5071         -  int i;
  5072         -
  5073         -  fts5BufferZero(pBuf);
  5074         -  for(i=0; i<pColset->nCol; i++){
  5075         -    const u8 *pSub = pPos;
  5076         -    int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
  5077         -    if( nSub ){
  5078         -      fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
  5079         -    }
  5080         -  }
  5081         -  return rc;
  5082         -}
  5083         -
  5084         -
  5085         -/*
  5086         -** Return a pointer to a buffer containing a copy of the position list for
  5087         -** the current entry. Output variable *pn is set to the size of the buffer 
  5088         -** in bytes before returning.
  5089         -**
  5090         -** The returned position list does not include the "number of bytes" varint
  5091         -** field that starts the position list on disk.
  5092         -*/
  5093         -int sqlite3Fts5IterPoslist(
  5094         -  Fts5IndexIter *pIter, 
  5095         -  Fts5Colset *pColset,            /* Column filter (or NULL) */
  5096         -  const u8 **pp,                  /* OUT: Pointer to position-list data */
  5097         -  int *pn,                        /* OUT: Size of position-list in bytes */
  5098         -  i64 *piRowid                    /* OUT: Current rowid */
  5099         -){
  5100         -  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  5101         -  int eDetail = pIter->pIndex->pConfig->eDetail;
  5102         -
  5103         -  assert( pIter->pIndex->rc==SQLITE_OK );
  5104         -  *piRowid = pSeg->iRowid;
  5105         -  if( eDetail==FTS5_DETAIL_NONE ){
  5106         -    *pn = pSeg->nPos;
  5107         -  }else
  5108         -  if( eDetail==FTS5_DETAIL_FULL 
  5109         -   && pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf 
  5110         -  ){
  5111         -    u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset];
  5112         -    if( pColset==0 || pIter->bFiltered ){
  5113         -      *pn = pSeg->nPos;
  5114         -      *pp = pPos;
  5115         -    }else if( pColset->nCol==1 ){
  5116         -      *pp = pPos;
  5117         -      *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]);
  5118         -    }else{
  5119         -      fts5BufferZero(&pIter->poslist);
  5120         -      fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist);
  5121         -      *pp = pIter->poslist.p;
  5122         -      *pn = pIter->poslist.n;
  5123         -    }
  5124         -  }else{
  5125         -    fts5BufferZero(&pIter->poslist);
  5126         -    fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist);
  5127         -    if( eDetail==FTS5_DETAIL_FULL ){
  5128         -      *pp = pIter->poslist.p;
  5129         -    }
  5130         -    *pn = pIter->poslist.n;
  5131         -  }
  5132         -  return fts5IndexReturn(pIter->pIndex);
  5133         -}
  5134         -
  5135         -int sqlite3Fts5IterCollist(
  5136         -  Fts5IndexIter *pIter, 
  5137         -  const u8 **pp,                  /* OUT: Pointer to position-list data */
  5138         -  int *pn                         /* OUT: Size of position-list in bytes */
  5139         -){
  5140         -  assert( pIter->pIndex->pConfig->eDetail==FTS5_DETAIL_COLUMNS );
  5141         -  *pp = pIter->poslist.p;
  5142         -  *pn = pIter->poslist.n;
  5143         -  return SQLITE_OK;
  5144         -}
  5145         -
  5146         -/*
  5147         -** This function is similar to sqlite3Fts5IterPoslist(), except that it
  5148         -** copies the position list into the buffer supplied as the second 
  5149         -** argument.
  5150         -*/
  5151         -int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){
  5152         -  Fts5Index *p = pIter->pIndex;
  5153         -  Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  5154         -  assert( p->rc==SQLITE_OK );
  5155         -  fts5BufferZero(pBuf);
  5156         -  fts5SegiterPoslist(p, pSeg, 0, pBuf);
  5157         -  return fts5IndexReturn(p);
  5158         -}
  5159         -
  5160   5271   /*
  5161   5272   ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery().
  5162   5273   */
  5163         -void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
  5164         -  if( pIter ){
         5274  +void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){
         5275  +  if( pIndexIter ){
         5276  +    Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
  5165   5277       Fts5Index *pIndex = pIter->pIndex;
  5166   5278       fts5MultiIterFree(pIter->pIndex, pIter);
  5167   5279       fts5CloseReader(pIndex);
  5168   5280     }
  5169   5281   }
  5170   5282   
  5171   5283   /*
................................................................................
  5324   5436     const char *z,                  /* Index key to query for */
  5325   5437     int n,                          /* Size of index key in bytes */
  5326   5438     int flags,                      /* Flags for Fts5IndexQuery */
  5327   5439     u64 *pCksum                     /* IN/OUT: Checksum value */
  5328   5440   ){
  5329   5441     int eDetail = p->pConfig->eDetail;
  5330   5442     u64 cksum = *pCksum;
  5331         -  Fts5IndexIter *pIdxIter = 0;
  5332         -  Fts5Buffer buf = {0, 0, 0};
  5333         -  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter);
         5443  +  Fts5IndexIter *pIter = 0;
         5444  +  int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter);
  5334   5445   
  5335         -  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  5336         -    i64 rowid = sqlite3Fts5IterRowid(pIdxIter);
         5446  +  while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){
         5447  +    i64 rowid = sqlite3Fts5IterRowid(pIter);
  5337   5448   
  5338   5449       if( eDetail==FTS5_DETAIL_NONE ){
  5339   5450         cksum ^= sqlite3Fts5IndexEntryCksum(rowid, 0, 0, iIdx, z, n);
  5340   5451       }else{
  5341         -      rc = sqlite3Fts5IterPoslistBuffer(pIdxIter, &buf);
  5342         -      if( rc==SQLITE_OK ){
  5343         -        Fts5PoslistReader sReader;
  5344         -        for(sqlite3Fts5PoslistReaderInit(buf.p, buf.n, &sReader);
  5345         -            sReader.bEof==0;
  5346         -            sqlite3Fts5PoslistReaderNext(&sReader)
  5347         -        ){
  5348         -          int iCol = FTS5_POS2COLUMN(sReader.iPos);
  5349         -          int iOff = FTS5_POS2OFFSET(sReader.iPos);
  5350         -          cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5351         -        }
         5452  +      Fts5PoslistReader sReader;
         5453  +      for(sqlite3Fts5PoslistReaderInit(pIter->pData, pIter->nData, &sReader);
         5454  +          sReader.bEof==0;
         5455  +          sqlite3Fts5PoslistReaderNext(&sReader)
         5456  +      ){
         5457  +        int iCol = FTS5_POS2COLUMN(sReader.iPos);
         5458  +        int iOff = FTS5_POS2OFFSET(sReader.iPos);
         5459  +        cksum ^= sqlite3Fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  5352   5460         }
  5353   5461       }
  5354   5462       if( rc==SQLITE_OK ){
  5355         -      rc = sqlite3Fts5IterNext(pIdxIter);
         5463  +      rc = sqlite3Fts5IterNext(pIter);
  5356   5464       }
  5357   5465     }
  5358         -  sqlite3Fts5IterClose(pIdxIter);
  5359         -  fts5BufferFree(&buf);
         5466  +  sqlite3Fts5IterClose(pIter);
  5360   5467   
  5361   5468     *pCksum = cksum;
  5362   5469     return rc;
  5363   5470   }
  5364   5471   
  5365   5472   
  5366   5473   /*
................................................................................
  5657   5764   ** error, or some other SQLite error code if another error (e.g. OOM)
  5658   5765   ** occurs.
  5659   5766   */
  5660   5767   int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  5661   5768     int eDetail = p->pConfig->eDetail;
  5662   5769     u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  5663   5770     Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
  5664         -  Fts5IndexIter *pIter;           /* Used to iterate through entire index */
         5771  +  Fts5Iter *pIter;                /* Used to iterate through entire index */
  5665   5772     Fts5Structure *pStruct;         /* Index structure */
  5666   5773   
  5667   5774   #ifdef SQLITE_DEBUG
  5668   5775     /* Used by extra internal tests only run if NDEBUG is not defined */
  5669   5776     u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
  5670   5777     Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
  5671   5778   #endif

Changes to ext/fts5/fts5_main.c.

   534    534     aColMap[1] = pConfig->nCol;
   535    535     aColMap[2] = pConfig->nCol+1;
   536    536   
   537    537     /* Set idxFlags flags for all WHERE clause terms that will be used. */
   538    538     for(i=0; i<pInfo->nConstraint; i++){
   539    539       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   540    540       int j;
   541         -    for(j=0; j<(int)ArraySize(aConstraint); j++){
          541  +    for(j=0; j<ArraySize(aConstraint); j++){
   542    542         struct Constraint *pC = &aConstraint[j];
   543    543         if( p->iColumn==aColMap[pC->iCol] && p->op & pC->op ){
   544    544           if( p->usable ){
   545    545             pC->iConsIndex = i;
   546    546             idxFlags |= pC->fts5op;
   547    547           }else if( j==0 ){
   548    548             /* As there exists an unusable MATCH constraint this is an 
................................................................................
   581    581       pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
   582    582     }else{
   583    583       pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
   584    584     }
   585    585   
   586    586     /* Assign argvIndex values to each constraint in use. */
   587    587     iNext = 1;
   588         -  for(i=0; i<(int)ArraySize(aConstraint); i++){
          588  +  for(i=0; i<ArraySize(aConstraint); i++){
   589    589       struct Constraint *pC = &aConstraint[i];
   590    590       if( pC->iConsIndex>=0 ){
   591    591         pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++;
   592    592         pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
   593    593       }
   594    594     }
   595    595   

Changes to ext/fts5/fts5_storage.c.

   334    334   */
   335    335   int sqlite3Fts5StorageClose(Fts5Storage *p){
   336    336     int rc = SQLITE_OK;
   337    337     if( p ){
   338    338       int i;
   339    339   
   340    340       /* Finalize all SQL statements */
   341         -    for(i=0; i<(int)ArraySize(p->aStmt); i++){
          341  +    for(i=0; i<ArraySize(p->aStmt); i++){
   342    342         sqlite3_finalize(p->aStmt[i]);
   343    343       }
   344    344   
   345    345       sqlite3_free(p);
   346    346     }
   347    347     return rc;
   348    348   }

Changes to ext/fts5/fts5_tokenize.c.

  1216   1216       { "ascii",     {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
  1217   1217       { "porter",    {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
  1218   1218     };
  1219   1219     
  1220   1220     int rc = SQLITE_OK;             /* Return code */
  1221   1221     int i;                          /* To iterate through builtin functions */
  1222   1222   
  1223         -  for(i=0; rc==SQLITE_OK && i<(int)ArraySize(aBuiltin); i++){
         1223  +  for(i=0; rc==SQLITE_OK && i<ArraySize(aBuiltin); i++){
  1224   1224       rc = pApi->xCreateTokenizer(pApi,
  1225   1225           aBuiltin[i].zName,
  1226   1226           (void*)pApi,
  1227   1227           &aBuiltin[i].x,
  1228   1228           0
  1229   1229       );
  1230   1230     }
  1231   1231   
  1232   1232     return rc;
  1233   1233   }
  1234   1234   
  1235   1235   

Changes to ext/fts5/fts5_vocab.c.

   180    180       const char *zType = bDb ? argv[5] : argv[4];
   181    181       int nDb = (int)strlen(zDb)+1; 
   182    182       int nTab = (int)strlen(zTab)+1;
   183    183       int eType = 0;
   184    184       
   185    185       rc = fts5VocabTableType(zType, pzErr, &eType);
   186    186       if( rc==SQLITE_OK ){
   187         -      assert( eType>=0 && eType<sizeof(azSchema)/sizeof(azSchema[0]) );
          187  +      assert( eType>=0 && eType<ArraySize(azSchema) );
   188    188         rc = sqlite3_declare_vtab(db, azSchema[eType]);
   189    189       }
   190    190   
   191    191       nByte = sizeof(Fts5VocabTable) + nDb + nTab;
   192    192       pRet = sqlite3Fts5MallocZero(&rc, nByte);
   193    193       if( pRet ){
   194    194         pRet->pGlobal = (Fts5Global*)pAux;
................................................................................
   403    403         sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm);
   404    404         memset(pCsr->aCnt, 0, nCol * sizeof(i64));
   405    405         memset(pCsr->aDoc, 0, nCol * sizeof(i64));
   406    406         pCsr->iCol = 0;
   407    407   
   408    408         assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW );
   409    409         while( rc==SQLITE_OK ){
   410         -        i64 dummy;
   411    410           const u8 *pPos; int nPos;   /* Position list */
   412    411           i64 iPos = 0;               /* 64-bit position read from poslist */
   413    412           int iOff = 0;               /* Current offset within position list */
   414    413   
          414  +        pPos = pCsr->pIter->pData;
          415  +        nPos = pCsr->pIter->nData;
   415    416           switch( pCsr->pConfig->eDetail ){
   416    417             case FTS5_DETAIL_FULL:
   417         -            rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
   418         -            if( rc==SQLITE_OK ){
   419         -              if( pTab->eType==FTS5_VOCAB_ROW ){
   420         -                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   421         -                  pCsr->aCnt[0]++;
   422         -                }
   423         -                pCsr->aDoc[0]++;
   424         -              }else{
   425         -                int iCol = -1;
   426         -                while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
   427         -                  int ii = FTS5_POS2COLUMN(iPos);
   428         -                  pCsr->aCnt[ii]++;
   429         -                  if( iCol!=ii ){
   430         -                    if( ii>=nCol ){
   431         -                      rc = FTS5_CORRUPT;
   432         -                      break;
   433         -                    }
   434         -                    pCsr->aDoc[ii]++;
   435         -                    iCol = ii;
          418  +            pPos = pCsr->pIter->pData;
          419  +            nPos = pCsr->pIter->nData;
          420  +            if( pTab->eType==FTS5_VOCAB_ROW ){
          421  +              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
          422  +                pCsr->aCnt[0]++;
          423  +              }
          424  +              pCsr->aDoc[0]++;
          425  +            }else{
          426  +              int iCol = -1;
          427  +              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
          428  +                int ii = FTS5_POS2COLUMN(iPos);
          429  +                pCsr->aCnt[ii]++;
          430  +                if( iCol!=ii ){
          431  +                  if( ii>=nCol ){
          432  +                    rc = FTS5_CORRUPT;
          433  +                    break;
   436    434                     }
          435  +                  pCsr->aDoc[ii]++;
          436  +                  iCol = ii;
   437    437                   }
   438    438                 }
   439    439               }
   440    440               break;
   441    441   
   442    442             case FTS5_DETAIL_COLUMNS:
   443    443               if( pTab->eType==FTS5_VOCAB_ROW ){
   444    444                 pCsr->aDoc[0]++;
   445    445               }else{
   446         -              Fts5Buffer buf = {0, 0, 0};
   447         -              rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
   448         -              if( rc==SQLITE_OK ){
   449         -                while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
   450         -                  assert_nc( iPos>=0 && iPos<nCol );
   451         -                  if( iPos>=nCol ){
   452         -                    rc = FTS5_CORRUPT;
   453         -                    break;
   454         -                  }
   455         -                  pCsr->aDoc[iPos]++;
          446  +              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){
          447  +                assert_nc( iPos>=0 && iPos<nCol );
          448  +                if( iPos>=nCol ){
          449  +                  rc = FTS5_CORRUPT;
          450  +                  break;
   456    451                   }
          452  +                pCsr->aDoc[iPos]++;
   457    453                 }
   458         -              sqlite3Fts5BufferFree(&buf);
   459    454               }
   460    455               break;
   461    456   
   462    457             default: 
   463    458               assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
   464    459               pCsr->aDoc[0]++;
   465    460               break;

Changes to ext/fts5/test/fts5_common.tcl.

    44     44   
    45     45     for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
    46     46       $cmd xPhraseForeach $i c o {
    47     47         lappend res $i.$c.$o
    48     48       }
    49     49     }
    50     50   
    51         -  set res
           51  +  #set res
           52  +  sort_poslist $res
    52     53   }
    53     54   
    54     55   proc fts5_test_collist {cmd} {
    55     56     set res [list]
    56     57   
    57     58     for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
    58     59       $cmd xPhraseColumnForeach $i c { lappend res $i.$c }

Changes to ext/fts5/test/fts5ac.test.

   154    154     do_execsql_test 1.$tn2.integrity {
   155    155       INSERT INTO xx(xx) VALUES('integrity-check');
   156    156     }
   157    157   
   158    158     #-------------------------------------------------------------------------
   159    159     #
   160    160     foreach {tn expr} {
   161         -    1.2 "a   OR b"
   162    161       1.1 "a   AND b"
          162  +    1.2 "a   OR b"
   163    163       1.3 "o"
   164    164       1.4 "b q"
   165    165       1.5 "e a e"
   166    166       1.6 "m d g q q b k b w f q q p p"
   167    167       1.7 "l o o l v v k"
   168    168       1.8 "a"
   169    169       1.9 "b"
................................................................................
   245    245       }
   246    246   
   247    247       set res [fts5_query_data $expr xx]
   248    248       do_execsql_test 1.$tn2.$tn.[llength $res].asc {
   249    249         SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
   250    250         FROM xx WHERE xx match $expr
   251    251       } $res
   252         -
   253    252   
   254    253       set res [fts5_query_data $expr xx DESC]
   255    254       do_execsql_test 1.$tn2.$tn.[llength $res].desc {
   256    255         SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx) 
   257    256         FROM xx WHERE xx match $expr ORDER BY 1 DESC
   258    257       } $res
   259    258     }

Added ext/fts5/test/fts5simple3.test.

            1  +# 2015 September 05
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +#
           12  +
           13  +source [file join [file dirname [info script]] fts5_common.tcl]
           14  +set testprefix fts5simple3
           15  +
           16  +# If SQLITE_ENABLE_FTS5 is defined, omit this file.
           17  +ifcapable !fts5 {
           18  +  finish_test
           19  +  return
           20  +}
           21  +
           22  +fts5_aux_test_functions db
           23  +
           24  +do_execsql_test 1.0 {
           25  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
           26  +  INSERT INTO t1 VALUES('a', 'b', 'c');
           27  +  INSERT INTO t1 VALUES('x', 'x', 'x');
           28  +}
           29  +
           30  +do_execsql_test 1.1 {
           31  +  SELECT rowid, fts5_test_collist(t1) FROM t1('a:a');
           32  +} {1 0.0}
           33  +
           34  +do_execsql_test 1.2 {
           35  +  SELECT rowid, fts5_test_collist(t1) FROM t1('b:x');
           36  +} {2 0.1}
           37  +
           38  +do_execsql_test 1.3 {
           39  +  SELECT rowid, fts5_test_collist(t1) FROM t1('b:a');
           40  +} {}
           41  +
           42  +
           43  +finish_test
           44  +

Changes to ext/fts5/test/fts5synonym2.test.

    23     23   
    24     24   foreach tok {query document} {
    25     25   foreach_detail_mode $testprefix {
    26     26   
    27     27   fts5_tclnum_register db
    28     28   fts5_aux_test_functions db
    29     29   
           30  +proc fts5_test_bothlist {cmd} {
           31  +
           32  +  for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
           33  +    set bFirst 1
           34  +    $cmd xPhraseColumnForeach $i c { 
           35  +      lappend CL $i.$c 
           36  +      if {$bFirst} { $cmd xPhraseForeach $i c o { lappend PL $i.$c.$o } }
           37  +      set bFirst 0
           38  +    }
           39  +  }
           40  +
           41  +  list [sort_poslist $PL] $CL
           42  +}
           43  +sqlite3_fts5_create_function db fts5_test_bothlist fts5_test_bothlist
           44  +
    30     45   proc fts5_rowid {cmd} { expr [$cmd xColumnText -1] }
    31     46   sqlite3_fts5_create_function db fts5_rowid fts5_rowid
    32     47   
    33     48   do_execsql_test 1.$tok.0.1 "
    34     49     CREATE VIRTUAL TABLE ss USING fts5(a, b, 
    35     50          tokenize='tclnum $tok', detail=%DETAIL%);
    36     51     INSERT INTO ss(ss, rank) VALUES('rank', 'fts5_rowid()');
................................................................................
    85    100     INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii');
    86    101     INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4');
    87    102     INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii');
    88    103     INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one');
    89    104   }
    90    105   
    91    106   foreach {tn expr} {
          107  +  2.1 "one OR two OR three OR four"
          108  +
    92    109     1.1 "one"   1.2 "two"   1.3 "three"   1.4 "four"
    93    110     1.5 "v"     1.6 "vi"    1.7 "vii"     1.8 "viii"
    94    111     1.9 "9"    1.10 "0"    1.11 "1"      1.12 "2"
    95    112   
    96    113     2.1 "one OR two OR three OR four"
    97    114     2.2 "(one AND two) OR (three AND four)"
    98    115     2.3 "(one AND two) OR (three AND four) NOT five"
................................................................................
   109    126     if {[fts5_expr_ok $expr ss]==0} {
   110    127       do_test 1.$tok.$tn.OMITTED { list } [list]
   111    128       continue
   112    129     }
   113    130   
   114    131     set res [fts5_query_data $expr ss ASC ::tclnum_syn]
   115    132     do_execsql_test 1.$tok.$tn.[llength $res].asc.1 {
   116         -    SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
          133  +    SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr)
   117    134     } $res
   118    135   
   119    136     do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
   120    137       SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
          138  +  } $res
          139  +
          140  +  do_execsql_test 1.$tok.$tn.[llength $res].asc.2 {
          141  +    SELECT rowid, fts5_test_poslist2(ss), fts5_test_collist(ss) FROM ss($expr)
   121    142       ORDER BY rank ASC
   122    143     } $res
          144  +
          145  +  set res2 [list]
          146  +  foreach {a b c} $res { lappend res2 $a $c $b }
          147  +  do_execsql_test 1.$tok.$tn.[llength $res].asc.3 {
          148  +    SELECT rowid, fts5_test_collist(ss), fts5_test_poslist2(ss) FROM ss($expr)
          149  +  } $res2
          150  +
          151  +  set res3 [list]
          152  +  foreach {a b c} $res { lappend res3 $a [list $b $c] }
          153  +  do_execsql_test 1.$tok.$tn.[llength $res].asc.3 {
          154  +    SELECT rowid, fts5_test_bothlist(ss) FROM ss($expr)
          155  +  } $res3
          156  +
          157  +
   123    158   }
   124    159   
   125    160   }
   126    161   }
   127    162   
   128    163   finish_test
   129    164   

Changes to ext/fts5/tool/fts5speed.tcl.

     7      7     {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"}
     8      8     {100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"}
     9      9     {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"}
    10     10   
    11     11     {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"}
    12     12     {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"}
    13     13     {1   "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"}
           14  +
           15  +  {2   "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:the'"}
    14     16   }
    15     17   
    16     18   proc usage {} {
    17     19     global Q
    18     20     puts stderr "Usage: $::argv0 DATABASE QUERY"
    19     21     puts stderr ""
    20     22     for {set i 1} {$i <= [llength $Q]} {incr i} {

Changes to ext/fts5/tool/fts5txt2db.tcl.

     1      1   
     2      2   
     3         -proc usage {} {
     4         -  puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..."
            3  +#-------------------------------------------------------------------------
            4  +# Command line options processor.
            5  +#
            6  +proc command_line_error {O E {msg ""}} {
            7  +  if {$msg != ""} {
            8  +    puts stderr "Error: $msg"
            9  +    puts stderr ""
           10  +  }
           11  +
           12  +  set L [list]
           13  +  foreach o $O {
           14  +    if {[llength $o]==1} {
           15  +      lappend L [string toupper $o]
           16  +    }
           17  +  }
           18  +
           19  +  puts stderr "Usage: $::argv0 ?SWITCHES? $L"
           20  +  puts stderr ""
           21  +  puts stderr "Switches are:"
           22  +  foreach o $O {
           23  +    if {[llength $o]==3} {
           24  +      foreach {a b c} $o {}
           25  +      puts stderr [format "    -%-15s %s (default \"%s\")" "$a VAL" $c $b]
           26  +    } elseif {[llength $o]==2} {
           27  +      foreach {a b} $o {}
           28  +      puts stderr [format "    -%-15s %s" $a $b]
           29  +    }
           30  +  }
     5     31     puts stderr ""
     6         -  puts stderr "Options are"
     7         -  puts stderr "  -fts5"
     8         -  puts stderr "  -fts4"
     9         -  puts stderr "  -colsize <list of column sizes>"
    10         -  puts stderr {
           32  +  puts stderr $E
           33  +  exit -1
           34  +}
           35  +
           36  +proc process_command_line {avar lArgs O E} {
           37  +
           38  +  upvar $avar A
           39  +  set zTrailing ""       ;# True if ... is present in $O
           40  +  set lPosargs [list]
           41  +
           42  +  # Populate A() with default values. Also, for each switch in the command
           43  +  # line spec, set an entry in the idx() array as follows:
           44  +  #
           45  +  #  {tblname t1 "table name to use"}  
           46  +  #      -> [set idx(-tblname) {tblname t1 "table name to use"}  
           47  +  #
           48  +  # For each position parameter, append its name to $lPosargs. If the ...
           49  +  # specifier is present, set $zTrailing to the name of the prefix.
           50  +  #
           51  +  foreach o $O {
           52  +    set nm [lindex $o 0]
           53  +    set nArg [llength $o]
           54  +    switch -- $nArg {
           55  +      1 {
           56  +        if {[string range $nm end-2 end]=="..."} {
           57  +          set zTrailing [string range $nm 0 end-3]
           58  +        } else {
           59  +          lappend lPosargs $nm
           60  +        }
           61  +      }
           62  +      2 {
           63  +        set A($nm) 0
           64  +        set idx(-$nm) $o
           65  +      }
           66  +      3 {
           67  +        set A($nm) [lindex $o 1]
           68  +        set idx(-$nm) $o
           69  +      }
           70  +      default {
           71  +        error "Error in command line specification"
           72  +      }
           73  +    }
           74  +  }
           75  +
           76  +  # Set explicitly specified option values
           77  +  #
           78  +  set nArg [llength $lArgs]
           79  +  for {set i 0} {$i < $nArg} {incr i} {
           80  +    set opt [lindex $lArgs $i]
           81  +    if {[string range $opt 0 0]!="-" || $opt=="--"} break
           82  +    set c [array names idx "${opt}*"]
           83  +    if {[llength $c]==0} { command_line_error $O $E "Unrecognized option: $opt"}
           84  +    if {[llength $c]>1}  { command_line_error $O $E "Ambiguous option: $opt"}
           85  +
           86  +    if {[llength $idx($c)]==3} {
           87  +      if {$i==[llength $lArgs]-1} {
           88  +        command_line_error $O $E "Option requires argument: $c" 
           89  +      }
           90  +      incr i
           91  +      set A([lindex $idx($c) 0]) [lindex $lArgs $i]
           92  +    } else {
           93  +      set A([lindex $idx($c) 0]) 1
           94  +    }
           95  +  }
           96  +
           97  +  # Deal with position arguments.
           98  +  #
           99  +  set nPosarg [llength $lPosargs]
          100  +  set nRem [expr $nArg - $i]
          101  +  if {$nRem < $nPosarg || ($zTrailing=="" && $nRem > $nPosarg)} {
          102  +    command_line_error $O $E
          103  +  }
          104  +  for {set j 0} {$j < $nPosarg} {incr j} {
          105  +    set A([lindex $lPosargs $j]) [lindex $lArgs [expr $j+$i]]
          106  +  }
          107  +  if {$zTrailing!=""} {
          108  +    set A($zTrailing) [lrange $lArgs [expr $j+$i] end]
          109  +  }
          110  +}
          111  +# End of command line options processor.
          112  +#-------------------------------------------------------------------------
          113  +
          114  +
          115  +process_command_line A $argv {
          116  +  {fts5                 "use fts5"}
          117  +  {fts4                 "use fts4"}
          118  +  {colsize   "10 10 10" "list of column sizes"}
          119  +  {tblname   "t1"       "table name to create"}
          120  +  {detail    "full"     "Fts5 detail mode to use"}
          121  +  {repeat    1          "Load each file this many times"}
          122  +  database
          123  +  file...
          124  +} {
    11    125   This script is designed to create fts4/5 tables with more than one column.
    12    126   The -colsize option should be set to a Tcl list of integer values, one for
    13    127   each column in the table. Each value is the number of tokens that will be
    14    128   inserted into the column value for each row. For example, setting the -colsize
    15    129   option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10
    16    130   tokens per row in each, respectively.
    17    131   
................................................................................
    18    132   Each "FILE" argument should be a text file. The contents of these text files is
    19    133   split on whitespace characters to form a list of tokens. The first N1 tokens
    20    134   are used for the first column of the first row, where N1 is the first element
    21    135   of the -colsize list. The next N2 are used for the second column of the first
    22    136   row, and so on. Rows are added to the table until the entire list of tokens
    23    137   is exhausted.
    24    138   }
    25         -  exit -1
          139  +
          140  +if {$A(fts4)} {
          141  +  set A(fts) fts4
          142  +} else {
          143  +  set A(fts) fts5
    26    144   }
    27    145   
    28         -set O(aColSize)       [list 10 10 10]
    29         -set O(tblname)        t1
    30         -set O(fts)            fts5
    31         -
    32         -
    33         -set options_with_values {-colsize}
    34         -
    35         -for {set i 0} {$i < [llength $argv]} {incr i} {
    36         -  set opt [lindex $argv $i]
    37         -  if {[string range $opt 0 0]!="-"} break
    38         -
    39         -  if {[lsearch $options_with_values $opt]>=0} {
    40         -    incr i
    41         -    if {$i==[llength $argv]} usage
    42         -    set val [lindex $argv $i]
    43         -  }
    44         -
    45         -  switch -- $opt {
    46         -    -colsize {
    47         -      set O(aColSize) $val
    48         -    }
    49         -
    50         -    -fts4 {
    51         -      set O(fts) fts4
    52         -    }
    53         -
    54         -    -fts5 {
    55         -      set O(fts) fts5
    56         -    }
    57         -  }
    58         -}
    59         -
    60         -if {$i > [llength $argv]-2} usage
    61         -set O(db) [lindex $argv $i]
    62         -set O(files) [lrange $argv [expr $i+1] end]
    63         -
    64         -sqlite3 db $O(db)
          146  +sqlite3 db $A(database)
    65    147   
    66    148   # Create the FTS table in the db. Return a list of the table columns.
    67    149   #
    68    150   proc create_table {} {
    69         -  global O
          151  +  global A
    70    152     set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z]
    71    153   
    72         -  set nCol [llength $O(aColSize)]
          154  +  set nCol [llength $A(colsize)]
    73    155     set cols [lrange $cols 0 [expr $nCol-1]]
    74    156   
    75         -  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) ("
          157  +  set sql    "CREATE VIRTUAL TABLE IF NOT EXISTS $A(tblname) USING $A(fts) ("
    76    158     append sql [join $cols ,]
    77         -  append sql ");"
          159  +  if {$A(fts)=="fts5"} { append sql ",detail=$A(detail));" }
    78    160   
    79    161     db eval $sql
    80    162     return $cols
    81    163   }
    82    164   
    83    165   # Return a list of tokens from the named file.
    84    166   #
................................................................................
    85    167   proc readfile {file} {
    86    168     set fd [open $file]
    87    169     set data [read $fd]
    88    170     close $fd
    89    171     split $data
    90    172   }
    91    173   
          174  +proc repeat {L n} {
          175  +  set res [list]
          176  +  for {set i 0} {$i < $n} {incr i} {
          177  +    set res [concat $res $L]
          178  +  }
          179  +  set res
          180  +}
          181  +
    92    182   
    93    183   # Load all the data into a big list of tokens.
    94    184   #
    95    185   set tokens [list]
    96         -foreach f $O(files) {
    97         -  set tokens [concat $tokens [readfile $f]]
          186  +foreach f $A(file) {
          187  +  set tokens [concat $tokens [repeat [readfile $f] $A(repeat)]]
    98    188   }
    99    189   
   100    190   set N [llength $tokens]
   101    191   set i 0
   102    192   set cols [create_table]
   103         -set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]"
          193  +set sql "INSERT INTO $A(tblname) VALUES(\$R([lindex $cols 0])"
   104    194   foreach c [lrange $cols 1 end] {
   105         -  append sql ", \$A($c)"
          195  +  append sql ", \$R($c)"
   106    196   }
   107    197   append sql ")"
   108    198   
   109    199   db eval BEGIN
   110    200     while {$i < $N} {
   111         -    foreach c $cols s $O(aColSize) {
   112         -      set A($c) [lrange $tokens $i [expr $i+$s-1]]
          201  +    foreach c $cols s $A(colsize) {
          202  +      set R($c) [lrange $tokens $i [expr $i+$s-1]]
   113    203         incr i $s
   114    204       }
   115    205       db eval $sql
   116    206     }
   117    207   db eval COMMIT
   118    208   
   119    209   
   120    210