/ Check-in [a684b5e2d9]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change to storing all keys in a single merge-tree structure instead of one main structure and a separate one for each prefix index. This is a file-format change. Also introduce a mechanism for managing file-format changes.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: a684b5e2d9d52cf4700e7e5f9dd547a2ba54e8e9
User & Date: dan 2015-05-07 19:29:46
Context
2015-05-08
09:21
Improve the error message returned by FTS5 if it encounters an unknown file format. check-in: f369caec14 user: dan tags: fts5
2015-05-07
19:29
Change to storing all keys in a single merge-tree structure instead of one main structure and a separate one for each prefix index. This is a file-format change. Also introduce a mechanism for managing file-format changes. check-in: a684b5e2d9 user: dan tags: fts5
2015-05-02
20:35
Reorganize some of the fts5 expression parsing code. Improve test coverage of the same. check-in: c4456dc5f5 user: dan tags: fts5
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5.c.

  1160   1160       if( rc==SQLITE_OK ){
  1161   1161         rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError);
  1162   1162       }
  1163   1163       if( rc==SQLITE_OK ){
  1164   1164         if( bError ){
  1165   1165           rc = SQLITE_ERROR;
  1166   1166         }else{
  1167         -        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal);
         1167  +        rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal, 0);
  1168   1168         }
  1169   1169       }
  1170   1170     }
  1171   1171     return rc;
  1172   1172   }
  1173   1173   
  1174   1174   static int fts5SpecialDelete(

Changes to ext/fts5/fts5Int.h.

   117    117     int iCookie;                    /* Incremented when %_config is modified */
   118    118     int pgsz;                       /* Approximate page size used in %_data */
   119    119     int nAutomerge;                 /* 'automerge' setting */
   120    120     int nCrisisMerge;               /* Maximum allowed segments per level */
   121    121     char *zRank;                    /* Name of rank function */
   122    122     char *zRankArgs;                /* Arguments to rank function */
   123    123   };
          124  +
          125  +/* Current expected value of %_config table 'version' field */
          126  +#define FTS5_CURRENT_VERSION 1
   124    127   
   125    128   #define FTS5_CONTENT_NORMAL   0
   126    129   #define FTS5_CONTENT_NONE     1
   127    130   #define FTS5_CONTENT_EXTERNAL 2
          131  +
   128    132   
   129    133   
   130    134   
   131    135   int sqlite3Fts5ConfigParse(
   132    136       Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
   133    137   );
   134    138   void sqlite3Fts5ConfigFree(Fts5Config*);
................................................................................
   390    394   void sqlite3Fts5HashFree(Fts5Hash*);
   391    395   
   392    396   int sqlite3Fts5HashWrite(
   393    397     Fts5Hash*,
   394    398     i64 iRowid,                     /* Rowid for this entry */
   395    399     int iCol,                       /* Column token appears in (-ve -> delete) */
   396    400     int iPos,                       /* Position of token within column */
          401  +  char bByte,
   397    402     const char *pToken, int nToken  /* Token to add or remove to or from index */
   398    403   );
   399    404   
   400    405   /*
   401    406   ** Empty (but do not delete) a hash table.
   402    407   */
   403    408   void sqlite3Fts5HashClear(Fts5Hash*);
................................................................................
   454    459   int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol);
   455    460   int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg);
   456    461   int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow);
   457    462   
   458    463   int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit);
   459    464   int sqlite3Fts5StorageRollback(Fts5Storage *p);
   460    465   
   461         -int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*);
          466  +int sqlite3Fts5StorageConfigValue(
          467  +    Fts5Storage *p, const char*, sqlite3_value*, int
          468  +);
   462    469   
   463    470   int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**);
   464    471   
   465    472   int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
   466    473   int sqlite3Fts5StorageRebuild(Fts5Storage *p);
   467    474   int sqlite3Fts5StorageOptimize(Fts5Storage *p);
   468    475   int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);

Changes to ext/fts5/fts5_config.c.

   199    199     assert( 0==fts5_iswhitespace(z[0]) );
   200    200     quote = z[0];
   201    201     if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){
   202    202       fts5Dequote(z);
   203    203     }
   204    204   }
   205    205   
   206         -/*
   207         -** Argument z points to a nul-terminated string containing an SQL identifier.
   208         -** This function returns a copy of the identifier enclosed in backtick 
   209         -** quotes.
   210         -*/
   211         -static char *fts5EscapeName(int *pRc, const char *z){
   212         -  char *pRet = 0;
   213         -  if( *pRc==SQLITE_OK ){
   214         -    int n = strlen(z);
   215         -    pRet = (char*)sqlite3_malloc(2 + 2*n + 1);
   216         -    if( pRet==0 ){
   217         -      *pRc = SQLITE_NOMEM;
   218         -    }else{
   219         -      int i;
   220         -      char *p = pRet;
   221         -      *p++ = '`';
   222         -      for(i=0; i<n; i++){
   223         -        if( z[i]=='`' ) *p++ = '`';
   224         -        *p++ = z[i];
   225         -      }
   226         -      *p++ = '`';
   227         -      *p++ = '\0';
   228         -    }
   229         -  }
   230         -  return pRet;
   231         -}
   232         -
   233    206   /*
   234    207   ** Parse the "special" CREATE VIRTUAL TABLE directive and update
   235    208   ** configuration object pConfig as appropriate.
   236    209   **
   237    210   ** If successful, object pConfig is updated and SQLITE_OK returned. If
   238    211   ** an error occurs, an SQLite error code is returned and an error message
   239    212   ** may be left in *pzErr. It is the responsibility of the caller to
................................................................................
   455    428   /*
   456    429   ** Populate the Fts5Config.zContentExprlist string.
   457    430   */
   458    431   static int fts5ConfigMakeExprlist(Fts5Config *p){
   459    432     int i;
   460    433     int rc = SQLITE_OK;
   461    434     Fts5Buffer buf = {0, 0, 0};
   462         -  const char *zSep = "";
   463    435   
   464    436     sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
   465    437     if( p->eContent!=FTS5_CONTENT_NONE ){
   466    438       for(i=0; i<p->nCol; i++){
   467    439         if( p->eContent==FTS5_CONTENT_EXTERNAL ){
   468    440           sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
   469    441         }else{
................................................................................
   845    817   ** Load the contents of the %_config table into memory.
   846    818   */
   847    819   int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   848    820     const char *zSelect = "SELECT k, v FROM %Q.'%q_config'";
   849    821     char *zSql;
   850    822     sqlite3_stmt *p = 0;
   851    823     int rc;
          824  +  int iVersion = 0;
   852    825   
   853    826     /* Set default values */
   854    827     pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE;
   855    828     pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE;
   856    829     pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   857    830   
   858    831     zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName);
................................................................................
   864    837     }
   865    838   
   866    839     assert( rc==SQLITE_OK || p==0 );
   867    840     if( rc==SQLITE_OK ){
   868    841       while( SQLITE_ROW==sqlite3_step(p) ){
   869    842         const char *zK = (const char*)sqlite3_column_text(p, 0);
   870    843         sqlite3_value *pVal = sqlite3_column_value(p, 1);
   871         -      sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
          844  +      if( 0==sqlite3_stricmp(zK, "version") ){
          845  +        iVersion = sqlite3_value_int(pVal);
          846  +      }else{
          847  +        sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0);
          848  +      }
   872    849       }
   873         -    rc = sqlite3_finalize(p);
          850  +    if( rc==SQLITE_OK ) rc = sqlite3_finalize(p);
          851  +  }
          852  +  
          853  +  if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
          854  +    rc = sqlite3Fts5Corrupt();
   874    855     }
   875    856   
   876    857     if( rc==SQLITE_OK ){
   877    858       pConfig->iCookie = iCookie;
   878    859     }
   879    860     return rc;
   880    861   }
   881    862   
   882    863   #endif /* SQLITE_ENABLE_FTS5 */

Changes to ext/fts5/fts5_expr.c.

  1367   1367   
  1368   1368   static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){
  1369   1369     char *zNew;
  1370   1370     va_list ap;
  1371   1371     va_start(ap, zFmt);
  1372   1372     zNew = sqlite3_vmprintf(zFmt, ap);
  1373   1373     va_end(ap);
  1374         -  if( zApp ){
         1374  +  if( zApp && zNew ){
  1375   1375       char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew);
  1376   1376       sqlite3_free(zNew);
  1377   1377       zNew = zNew2;
  1378   1378     }
  1379   1379     sqlite3_free(zApp);
  1380   1380     return zNew;
  1381   1381   }
................................................................................
  1544   1544     int rc;
  1545   1545     int i;
  1546   1546   
  1547   1547     const char **azConfig;          /* Array of arguments for Fts5Config */
  1548   1548     const char *zNearsetCmd = "nearset";
  1549   1549     int nConfig;                    /* Size of azConfig[] */
  1550   1550     Fts5Config *pConfig = 0;
         1551  +  int iArg = 1;
  1551   1552   
  1552   1553     if( bTcl && nArg>1 ){
  1553   1554       zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]);
         1555  +    iArg = 2;
  1554   1556     }
  1555   1557   
  1556         -  nConfig = nArg + 2 - bTcl;
         1558  +  nConfig = 3 + (nArg-iArg);
  1557   1559     azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig);
  1558   1560     if( azConfig==0 ){
  1559   1561       sqlite3_result_error_nomem(pCtx);
  1560   1562       return;
  1561   1563     }
  1562   1564     azConfig[0] = 0;
  1563   1565     azConfig[1] = "main";
  1564   1566     azConfig[2] = "tbl";
  1565         -  for(i=1+bTcl; i<nArg; i++){
  1566         -    azConfig[i+2-bTcl] = (const char*)sqlite3_value_text(apVal[i]);
         1567  +  for(i=3; iArg<nArg; iArg++){
         1568  +    azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]);
  1567   1569     }
         1570  +
  1568   1571     zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1569   1572   
  1570   1573     rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr);
  1571   1574     if( rc==SQLITE_OK ){
  1572   1575       rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr);
  1573   1576     }
  1574   1577     if( rc==SQLITE_OK ){
................................................................................
  1576   1579       if( pExpr->pRoot==0 ){
  1577   1580         zText = sqlite3_mprintf("");
  1578   1581       }else if( bTcl ){
  1579   1582         zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot);
  1580   1583       }else{
  1581   1584         zText = fts5ExprPrint(pConfig, pExpr->pRoot);
  1582   1585       }
  1583         -    if( rc==SQLITE_OK ){
         1586  +    if( zText==0 ){
         1587  +      rc = SQLITE_NOMEM;
         1588  +    }else{
  1584   1589         sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT);
  1585   1590         sqlite3_free(zText);
  1586   1591       }
  1587   1592     }
  1588   1593   
  1589   1594     if( rc!=SQLITE_OK ){
  1590   1595       if( zErr ){

Changes to ext/fts5/fts5_hash.c.

   131    131     int i;
   132    132     unsigned int h = 13;
   133    133     for(i=n-1; i>=0; i--){
   134    134       h = (h << 3) ^ h ^ p[i];
   135    135     }
   136    136     return (h % nSlot);
   137    137   }
          138  +
          139  +static unsigned int fts5HashKey2(int nSlot, char b, const char *p, int n){
          140  +  int i;
          141  +  unsigned int h = 13;
          142  +  for(i=n-1; i>=0; i--){
          143  +    h = (h << 3) ^ h ^ p[i];
          144  +  }
          145  +  h = (h << 3) ^ h ^ b;
          146  +  return (h % nSlot);
          147  +}
   138    148   
   139    149   /*
   140    150   ** Resize the hash table by doubling the number of slots.
   141    151   */
   142    152   static int fts5HashResize(Fts5Hash *pHash){
   143    153     int nNew = pHash->nSlot*2;
   144    154     int i;
................................................................................
   187    197   }
   188    198   
   189    199   int sqlite3Fts5HashWrite(
   190    200     Fts5Hash *pHash,
   191    201     i64 iRowid,                     /* Rowid for this entry */
   192    202     int iCol,                       /* Column token appears in (-ve -> delete) */
   193    203     int iPos,                       /* Position of token within column */
          204  +  char bByte,                     /* First byte of token */
   194    205     const char *pToken, int nToken  /* Token to add or remove to or from index */
   195    206   ){
   196         -  unsigned int iHash = fts5HashKey(pHash->nSlot, pToken, nToken);
          207  +  unsigned int iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken);
   197    208     Fts5HashEntry *p;
   198    209     u8 *pPtr;
   199    210     int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
   200    211   
   201    212     /* Attempt to locate an existing hash entry */
   202    213     for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
   203         -    if( memcmp(p->zKey, pToken, nToken)==0 && p->zKey[nToken]==0 ) break;
          214  +    if( p->zKey[0]==bByte 
          215  +     && memcmp(&p->zKey[1], pToken, nToken)==0 
          216  +     && p->zKey[nToken+1]==0 
          217  +    ){
          218  +      break;
          219  +    }
   204    220     }
   205    221   
   206    222     /* If an existing hash entry cannot be found, create a new one. */
   207    223     if( p==0 ){
   208         -    int nByte = sizeof(Fts5HashEntry) + nToken + 1 + 64;
          224  +    int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64;
   209    225       if( nByte<128 ) nByte = 128;
   210    226   
   211    227       if( (pHash->nEntry*2)>=pHash->nSlot ){
   212    228         int rc = fts5HashResize(pHash);
   213    229         if( rc!=SQLITE_OK ) return rc;
   214         -      iHash = fts5HashKey(pHash->nSlot, pToken, nToken);
          230  +      iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken);
   215    231       }
   216    232   
   217    233       p = (Fts5HashEntry*)sqlite3_malloc(nByte);
   218    234       if( !p ) return SQLITE_NOMEM;
   219    235       memset(p, 0, sizeof(Fts5HashEntry));
   220    236       p->nAlloc = nByte;
   221         -    memcpy(p->zKey, pToken, nToken);
   222         -    p->zKey[nToken] = '\0';
   223         -    p->nData = nToken + 1 + sizeof(Fts5HashEntry);
          237  +    p->zKey[0] = bByte;
          238  +    memcpy(&p->zKey[1], pToken, nToken);
          239  +    assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) );
          240  +    p->zKey[nToken+1] = '\0';
          241  +    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
   224    242       p->nData += sqlite3PutVarint(&((u8*)p)[p->nData], iRowid);
   225    243       p->iSzPoslist = p->nData;
   226    244       p->nData += 1;
   227    245       p->iRowid = iRowid;
   228    246       p->pHashNext = pHash->aSlot[iHash];
   229    247       pHash->aSlot[iHash] = p;
   230    248       pHash->nEntry++;

Changes to ext/fts5/fts5_index.c.

    45     45   
    46     46   
    47     47   #define FTS5_OPT_WORK_UNIT  1000  /* Number of leaf pages per optimize step */
    48     48   #define FTS5_WORK_UNIT      64    /* Number of leaf pages in unit of work */
    49     49   
    50     50   #define FTS5_MIN_DLIDX_SIZE 4     /* Add dlidx if this many empty pages */
    51     51   
           52  +#define FTS5_MAIN_PREFIX '0'
           53  +
           54  +#if FTS5_MAX_PREFIX_INDEXES > 31
           55  +# error "FTS5_MAX_PREFIX_INDEXES is too large"
           56  +#endif
           57  +
    52     58   /*
    53     59   ** Details:
    54     60   **
    55     61   ** The %_data table managed by this module,
    56     62   **
    57     63   **     CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB);
    58     64   **
................................................................................
   207    213   **   representing the first docid on the page otherwise.
   208    214   */
   209    215   
   210    216   /*
   211    217   ** Rowids for the averages and structure records in the %_data table.
   212    218   */
   213    219   #define FTS5_AVERAGES_ROWID     1    /* Rowid used for the averages record */
   214         -#define FTS5_STRUCTURE_ROWID(iIdx) (10 + (iIdx))     /* For structure records */
          220  +#define FTS5_STRUCTURE_ROWID   10    /* The structure record */
   215    221   
   216    222   /*
   217    223   ** Macros determining the rowids used by segment nodes. All nodes in all
   218    224   ** segments for all indexes (the regular FTS index and any prefix indexes)
   219    225   ** are stored in the %_data table with large positive rowids.
   220    226   **
   221    227   ** The %_data table may contain up to (1<<FTS5_SEGMENT_INDEX_BITS) 
................................................................................
   229    235   ** a nodes page number is always one more than its left sibling.
   230    236   **
   231    237   ** The rowid for a node is then found using the FTS5_SEGMENT_ROWID() macro
   232    238   ** below. The FTS5_SEGMENT_*_BITS macros define the number of bits used
   233    239   ** to encode the three FTS5_SEGMENT_ROWID() arguments. This module returns
   234    240   ** SQLITE_FULL and fails the current operation if they ever prove too small.
   235    241   */
   236         -#define FTS5_DATA_IDX_B     5     /* Max of 31 prefix indexes */
   237    242   #define FTS5_DATA_ID_B     16     /* Max seg id number 65535 */
   238    243   #define FTS5_DATA_HEIGHT_B  5     /* Max b-tree height of 32 */
   239    244   #define FTS5_DATA_PAGE_B   31     /* Max page number of 2147483648 */
   240    245   
   241         -#define FTS5_SEGMENT_ROWID(idx, segid, height, pgno) (                         \
   242         - ((i64)(idx)    << (FTS5_DATA_ID_B + FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \
          246  +#define FTS5_SEGMENT_ROWID(segid, height, pgno) (                         \
   243    247    ((i64)(segid)  << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) +                  \
   244    248    ((i64)(height) << (FTS5_DATA_PAGE_B)) +                                       \
   245    249    ((i64)(pgno))                                                                 \
   246    250   )
   247    251   
   248         -#if FTS5_MAX_PREFIX_INDEXES > ((1<<FTS5_DATA_IDX_B)-1) 
   249         -# error "FTS5_MAX_PREFIX_INDEXES is too large"
   250         -#endif
   251         -
   252    252   /*
   253    253   ** The height of segment b-trees is actually limited to one less than 
   254    254   ** (1<<HEIGHT_BITS). This is because the rowid address space for nodes
   255    255   ** with such a height is used by doclist indexes.
   256    256   */
   257    257   #define FTS5_SEGMENT_MAX_HEIGHT ((1 << FTS5_DATA_HEIGHT_B)-1)
   258    258   
................................................................................
   261    261   */
   262    262   #define FTS5_MAX_SEGMENT 2000
   263    263   
   264    264   /*
   265    265   ** The rowid for the doclist index associated with leaf page pgno of segment
   266    266   ** segid in index idx.
   267    267   */
   268         -#define FTS5_DOCLIST_IDX_ROWID(idx, segid, pgno) \
   269         -        FTS5_SEGMENT_ROWID(idx, segid, FTS5_SEGMENT_MAX_HEIGHT, pgno)
          268  +#define FTS5_DOCLIST_IDX_ROWID(segid, pgno) \
          269  +        FTS5_SEGMENT_ROWID(segid, FTS5_SEGMENT_MAX_HEIGHT, pgno)
   270    270   
   271    271   #ifdef SQLITE_DEBUG
   272    272   int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
   273    273   #endif
   274    274   
   275    275   
   276    276   /*
................................................................................
   310    310     char *zDataTbl;                 /* Name of %_data table */
   311    311     int nWorkUnit;                  /* Leaf pages in a "unit" of work */
   312    312   
   313    313     /*
   314    314     ** Variables related to the accumulation of tokens and doclists within the
   315    315     ** in-memory hash tables before they are flushed to disk.
   316    316     */
   317         -  Fts5Hash **apHash;              /* Array of hash tables */
          317  +  Fts5Hash *pHash;                /* Hash table for in-memory data */
   318    318     int nMaxPendingData;            /* Max pending data before flush to disk */
   319    319     int nPendingData;               /* Current bytes of pending data */
   320    320     i64 iWriteRowid;                /* Rowid for current doc being written */
          321  +  Fts5Buffer scratch;
   321    322   
   322    323     /* Error state. */
   323    324     int rc;                         /* Current error code */
   324    325   
   325    326     /* State used by the fts5DataXXX() functions. */
   326    327     sqlite3_blob *pReader;          /* RO incr-blob open on %_data table */
   327    328     sqlite3_stmt *pWriter;          /* "INSERT ... %_data VALUES(?,?)" */
................................................................................
   380    381   */
   381    382   struct Fts5PageWriter {
   382    383     int pgno;                       /* Page number for this page */
   383    384     Fts5Buffer buf;                 /* Buffer containing page data */
   384    385     Fts5Buffer term;                /* Buffer containing previous term on page */
   385    386   };
   386    387   struct Fts5SegWriter {
   387         -  int iIdx;                       /* Index to write to */
   388    388     int iSegid;                     /* Segid to write to */
   389    389     int nWriter;                    /* Number of entries in aWriter */
   390    390     Fts5PageWriter *aWriter;        /* Array of PageWriter objects */
   391    391     i64 iPrevRowid;                 /* Previous docid written to current leaf */
   392    392     u8 bFirstRowidInDoclist;        /* True if next rowid is first in doclist */
   393    393     u8 bFirstRowidInPage;           /* True if next rowid is first in page */
   394    394     u8 bFirstTermInPage;            /* True if next term will be first in leaf */
................................................................................
   474    474   **
   475    475   **     For each rowid on the page corresponding to the current term, the
   476    476   **     corresponding aRowidOffset[] entry is set to the byte offset of the
   477    477   **     start of the "position-list-size" field within the page.
   478    478   */
   479    479   struct Fts5SegIter {
   480    480     Fts5StructureSegment *pSeg;     /* Segment to iterate through */
   481         -  int iIdx;                       /* Byte offset within current leaf */
   482    481     int flags;                      /* Mask of configuration flags */
   483    482     int iLeafPgno;                  /* Current leaf page number */
   484    483     Fts5Data *pLeaf;                /* Current leaf data */
   485    484     int iLeafOffset;                /* Byte offset within current leaf */
   486    485   
   487    486     /* The page and offset from which the current term was read. The offset 
   488    487     ** is the offset of the first rowid in the current doclist.  */
................................................................................
   595    594   struct Fts5BtreeIterLevel {
   596    595     Fts5NodeIter s;                 /* Iterator for the current node */
   597    596     Fts5Data *pData;                /* Data for the current node */
   598    597   };
   599    598   struct Fts5BtreeIter {
   600    599     Fts5Index *p;                   /* FTS5 backend object */
   601    600     Fts5StructureSegment *pSeg;     /* Iterate through this segment's b-tree */
   602         -  int iIdx;                       /* Index pSeg belongs to */
   603    601     int nLvl;                       /* Size of aLvl[] array */
   604    602     Fts5BtreeIterLevel *aLvl;       /* Level for each tier of b-tree */
   605    603   
   606    604     /* Output variables */
   607    605     Fts5Buffer term;                /* Current term */
   608    606     int iLeaf;                      /* Leaf containing terms >= current term */
   609    607     int nEmpty;                     /* Number of "empty" leaves following iLeaf */
................................................................................
   987    985       sqlite3_blob_close(p->pReader);
   988    986       p->pReader = 0;
   989    987     }
   990    988   }
   991    989   #endif
   992    990   
   993    991   /*
   994         -** Remove all records associated with segment iSegid in index iIdx.
          992  +** Remove all records associated with segment iSegid.
   995    993   */
   996         -static void fts5DataRemoveSegment(Fts5Index *p, int iIdx, int iSegid){
   997         -  i64 iFirst = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, 0);
   998         -  i64 iLast = FTS5_SEGMENT_ROWID(iIdx, iSegid+1, 0, 0)-1;
          994  +static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){
          995  +  i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0);
          996  +  i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1;
   999    997     fts5DataDelete(p, iFirst, iLast);
  1000    998   }
  1001    999   
  1002   1000   /*
  1003   1001   ** Release a reference to an Fts5Structure object returned by an earlier 
  1004   1002   ** call to fts5StructureRead() or fts5StructureDecode().
  1005   1003   */
................................................................................
  1142   1140       }else{
  1143   1141         *pRc = SQLITE_NOMEM;
  1144   1142       }
  1145   1143     }
  1146   1144   }
  1147   1145   
  1148   1146   /*
  1149         -** Read, deserialize and return the structure record for index iIdx.
         1147  +** Read, deserialize and return the structure record.
  1150   1148   **
  1151   1149   ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array
  1152   1150   ** are over-allocated as described for function fts5StructureDecode() 
  1153   1151   ** above.
  1154   1152   **
  1155   1153   ** If an error occurs, NULL is returned and an error code left in the
  1156   1154   ** Fts5Index handle. If an error has already occurred when this function
  1157   1155   ** is called, it is a no-op.
  1158   1156   */
  1159         -static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){
         1157  +static Fts5Structure *fts5StructureRead(Fts5Index *p){
  1160   1158     Fts5Config *pConfig = p->pConfig;
  1161   1159     Fts5Structure *pRet = 0;        /* Object to return */
  1162   1160     Fts5Data *pData;                /* %_data entry containing structure record */
  1163   1161     int iCookie;                    /* Configuration cookie */
  1164   1162   
  1165         -  assert( iIdx<=pConfig->nPrefix );
  1166         -  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx));
         1163  +  pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
  1167   1164     if( !pData ) return 0;
  1168   1165     p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);
  1169   1166   
  1170   1167     if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
  1171   1168       p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
  1172   1169     }
  1173   1170   
................................................................................
  1194   1191     }
  1195   1192   
  1196   1193     return nSegment;
  1197   1194   }
  1198   1195   #endif
  1199   1196   
  1200   1197   /*
  1201         -** Serialize and store the "structure" record for index iIdx.
         1198  +** Serialize and store the "structure" record.
  1202   1199   **
  1203   1200   ** If an error occurs, leave an error code in the Fts5Index object. If an
  1204   1201   ** error has already occurred, this function is a no-op.
  1205   1202   */
  1206         -static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
         1203  +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){
  1207   1204     if( p->rc==SQLITE_OK ){
  1208   1205       Fts5Buffer buf;               /* Buffer to serialize record into */
  1209   1206       int iLvl;                     /* Used to iterate through levels */
  1210   1207       int iCookie;                  /* Cookie value to store */
  1211   1208   
  1212   1209       assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  1213   1210       memset(&buf, 0, sizeof(Fts5Buffer));
................................................................................
  1232   1229           fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid);
  1233   1230           fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nHeight);
  1234   1231           fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst);
  1235   1232           fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast);
  1236   1233         }
  1237   1234       }
  1238   1235   
  1239         -    fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n);
         1236  +    fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n);
  1240   1237       fts5BufferFree(&buf);
  1241   1238     }
  1242   1239   }
  1243   1240   
  1244   1241   #if 0
  1245   1242   static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*);
  1246   1243   static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){
................................................................................
  1528   1525   
  1529   1526     return pIter->bEof;
  1530   1527   }
  1531   1528   
  1532   1529   static Fts5DlidxIter *fts5DlidxIterInit(
  1533   1530     Fts5Index *p,                   /* Fts5 Backend to iterate within */
  1534   1531     int bRev,                       /* True for ORDER BY ASC */
  1535         -  int iIdx, int iSegid,           /* Segment iSegid within index iIdx */
         1532  +  int iSegid,                     /* Segment id */
  1536   1533     int iLeafPg                     /* Leaf page number to load dlidx for */
  1537   1534   ){
  1538   1535     Fts5DlidxIter *pIter;
  1539   1536   
  1540   1537     pIter = (Fts5DlidxIter*)fts5IdxMalloc(p, sizeof(Fts5DlidxIter));
  1541   1538     if( pIter==0 ) return 0;
  1542   1539   
  1543         -  pIter->pData = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iIdx, iSegid, iLeafPg));
         1540  +  pIter->pData = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iSegid, iLeafPg));
  1544   1541     if( pIter->pData==0 ){
  1545   1542       sqlite3_free(pIter);
  1546   1543       pIter = 0;
  1547   1544     }else{
  1548   1545       pIter->iLeafPgno = iLeafPg;
  1549   1546       if( bRev==0 ){
  1550   1547         fts5DlidxIterFirst(pIter);
................................................................................
  1579   1576     Fts5SegIter *pIter              /* Iterator to advance to next page */
  1580   1577   ){
  1581   1578     Fts5StructureSegment *pSeg = pIter->pSeg;
  1582   1579     fts5DataRelease(pIter->pLeaf);
  1583   1580     pIter->iLeafPgno++;
  1584   1581     if( pIter->iLeafPgno<=pSeg->pgnoLast ){
  1585   1582       pIter->pLeaf = fts5DataRead(p, 
  1586         -        FTS5_SEGMENT_ROWID(pIter->iIdx, pSeg->iSegid, 0, pIter->iLeafPgno)
         1583  +        FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pIter->iLeafPgno)
  1587   1584       );
  1588   1585     }else{
  1589   1586       pIter->pLeaf = 0;
  1590   1587     }
  1591   1588   }
  1592   1589   
  1593   1590   /*
................................................................................
  1665   1662     }
  1666   1663     iOff += sqlite3GetVarint(&a[iOff], (u64*)&pIter->iRowid);
  1667   1664     pIter->iLeafOffset = iOff;
  1668   1665   }
  1669   1666   
  1670   1667   /*
  1671   1668   ** Initialize the iterator object pIter to iterate through the entries in
  1672         -** segment pSeg within index iIdx. The iterator is left pointing to the 
  1673         -** first entry when this function returns.
         1669  +** segment pSeg. The iterator is left pointing to the first entry when 
         1670  +** this function returns.
  1674   1671   **
  1675   1672   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
  1676   1673   ** an error has already occurred when this function is called, it is a no-op.
  1677   1674   */
  1678   1675   static void fts5SegIterInit(
  1679         -  Fts5Index *p,          
  1680         -  int iIdx,                       /* Config.aHash[] index of FTS index */
         1676  +  Fts5Index *p,                   /* FTS index object */
  1681   1677     Fts5StructureSegment *pSeg,     /* Description of segment */
  1682   1678     Fts5SegIter *pIter              /* Object to populate */
  1683   1679   ){
  1684   1680     if( pSeg->pgnoFirst==0 ){
  1685   1681       /* This happens if the segment is being used as an input to an incremental
  1686   1682       ** merge and all data has already been "trimmed". See function
  1687   1683       ** fts5TrimSegments() for details. In this case leave the iterator empty.
................................................................................
  1690   1686       assert( pIter->pLeaf==0 );
  1691   1687       return;
  1692   1688     }
  1693   1689   
  1694   1690     if( p->rc==SQLITE_OK ){
  1695   1691       memset(pIter, 0, sizeof(*pIter));
  1696   1692       pIter->pSeg = pSeg;
  1697         -    pIter->iIdx = iIdx;
  1698   1693       pIter->iLeafPgno = pSeg->pgnoFirst-1;
  1699   1694       fts5SegIterNextPage(p, pIter);
  1700   1695     }
  1701   1696   
  1702   1697     if( p->rc==SQLITE_OK ){
  1703   1698       u8 *a = pIter->pLeaf->p;
  1704   1699       pIter->iLeafOffset = fts5GetU16(&a[2]);
................................................................................
  1767   1762   
  1768   1763     fts5DataRelease(pIter->pLeaf);
  1769   1764     pIter->pLeaf = 0;
  1770   1765     while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){
  1771   1766       Fts5Data *pNew;
  1772   1767       pIter->iLeafPgno--;
  1773   1768       pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
  1774         -          pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno
         1769  +          pIter->pSeg->iSegid, 0, pIter->iLeafPgno
  1775   1770       ));
  1776   1771       if( pNew ){
  1777   1772         if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
  1778   1773           if( pIter->iTermLeafOffset<pNew->n ){
  1779   1774             pIter->pLeaf = pNew;
  1780   1775             pIter->iLeafOffset = pIter->iTermLeafOffset;
  1781   1776           }
................................................................................
  1875   1870             pIter->iRowid += iDelta;
  1876   1871           }
  1877   1872         }else if( pIter->pSeg==0 ){
  1878   1873           const u8 *pList = 0;
  1879   1874           const char *zTerm;
  1880   1875           int nList;
  1881   1876           if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
  1882         -          sqlite3Fts5HashScanNext(p->apHash[0]);
  1883         -          sqlite3Fts5HashScanEntry(p->apHash[0], &zTerm, &pList, &nList);
         1877  +          sqlite3Fts5HashScanNext(p->pHash);
         1878  +          sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList);
  1884   1879           }
  1885   1880           if( pList==0 ){
  1886   1881             fts5DataRelease(pIter->pLeaf);
  1887   1882             pIter->pLeaf = 0;
  1888   1883           }else{
  1889   1884             pIter->pLeaf->p = (u8*)pList;
  1890   1885             pIter->pLeaf->n = nList;
................................................................................
  1931   1926   #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; }
  1932   1927   
  1933   1928   /*
  1934   1929   ** Iterator pIter currently points to the first rowid in a doclist. This
  1935   1930   ** function sets the iterator up so that iterates in reverse order through
  1936   1931   ** the doclist.
  1937   1932   */
  1938         -static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
         1933  +static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
  1939   1934     Fts5DlidxIter *pDlidx = pIter->pDlidx;
  1940   1935     Fts5Data *pLast = 0;
  1941   1936     int pgnoLast = 0;
  1942   1937   
  1943   1938     if( pDlidx ){
  1944   1939       /* If the doclist-iterator is already at EOF, then the current doclist
  1945   1940       ** contains no entries except those on the current page. */
  1946   1941       if( fts5DlidxIterEof(p, pDlidx)==0 ){
  1947   1942         int iSegid = pIter->pSeg->iSegid;
  1948   1943         pgnoLast = pDlidx->iLeafPgno;
  1949         -      pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pgnoLast));
         1944  +      pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast));
  1950   1945       }else{
  1951   1946         pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel);
  1952   1947       }
  1953   1948     }else{
  1954   1949       int iOff;                               /* Byte offset within pLeaf */
  1955   1950       Fts5Data *pLeaf = pIter->pLeaf;         /* Current leaf data */
  1956   1951   
................................................................................
  1985   1980       if( iOff>=pLeaf->n ){
  1986   1981         int pgno;
  1987   1982         Fts5StructureSegment *pSeg = pIter->pSeg;
  1988   1983   
  1989   1984         /* The last rowid in the doclist may not be on the current page. Search
  1990   1985         ** forward to find the page containing the last rowid.  */
  1991   1986         for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){
  1992         -        i64 iAbs = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pgno);
         1987  +        i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pgno);
  1993   1988           Fts5Data *pNew = fts5DataRead(p, iAbs);
  1994   1989           if( pNew ){
  1995   1990             int iRowid, iTerm;
  1996   1991             fts5LeafHeader(pNew, &iRowid, &iTerm);
  1997   1992             if( iRowid ){
  1998   1993               SWAPVAL(Fts5Data*, pNew, pLast);
  1999   1994               pgnoLast = pgno;
................................................................................
  2025   2020       pIter->iLeafOffset = iOff;
  2026   2021     }
  2027   2022   
  2028   2023     fts5SegIterReverseInitPage(p, pIter);
  2029   2024   }
  2030   2025   
  2031   2026   /*
  2032         -** Iterator pIter currently points to the first rowid of a doclist within
  2033         -** index iIdx. There is a doclist-index associated with the final term on
  2034         -** the current page. If the current term is the last term on the page, 
  2035         -** load the doclist-index from disk and initialize an iterator at 
  2036         -** (pIter->pDlidx).
         2027  +** Iterator pIter currently points to the first rowid of a doclist.
         2028  +** There is a doclist-index associated with the final term on the current 
         2029  +** page. If the current term is the last term on the page, load the 
         2030  +** doclist-index from disk and initialize an iterator at (pIter->pDlidx).
  2037   2031   */
  2038         -static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
         2032  +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
  2039   2033     int iSeg = pIter->pSeg->iSegid;
  2040   2034     int bRev = (pIter->flags & FTS5_SEGITER_REVERSE);
  2041   2035     Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */
  2042   2036   
  2043   2037     assert( pIter->flags & FTS5_SEGITER_ONETERM );
  2044   2038     assert( pIter->pDlidx==0 );
  2045   2039   
................................................................................
  2058   2052         if( iDelta==0 ) return;
  2059   2053         assert_nc( iOff<pLeaf->n );
  2060   2054         iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy);
  2061   2055         iOff += nPos;
  2062   2056       }
  2063   2057     }
  2064   2058   
  2065         -  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iIdx, iSeg, pIter->iTermLeafPgno);
         2059  +  pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
  2066   2060   }
  2067   2061   
  2068   2062   /*
  2069   2063   ** Initialize the object pIter to point to term pTerm/nTerm within segment
  2070         -** pSeg, index iIdx. If there is no such term in the index, the iterator
  2071         -** is set to EOF.
         2064  +** pSeg. If there is no such term in the index, the iterator is set to EOF.
  2072   2065   **
  2073   2066   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
  2074   2067   ** an error has already occurred when this function is called, it is a no-op.
  2075   2068   */
  2076   2069   static void fts5SegIterSeekInit(
  2077   2070     Fts5Index *p,                   /* FTS5 backend */
  2078         -  int iIdx,                       /* Config.aHash[] index of FTS index */
  2079   2071     const u8 *pTerm, int nTerm,     /* Term to seek to */
  2080   2072     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  2081   2073     Fts5StructureSegment *pSeg,     /* Description of segment */
  2082   2074     Fts5SegIter *pIter              /* Object to populate */
  2083   2075   ){
  2084   2076     int iPg = 1;
  2085   2077     int h;
  2086         -  int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0);
         2078  +  int bGe = (flags & FTS5INDEX_QUERY_PREFIX);
  2087   2079     int bDlidx = 0;                 /* True if there is a doclist-index */
  2088   2080   
  2089   2081     assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
  2090   2082     assert( pTerm && nTerm );
  2091   2083     memset(pIter, 0, sizeof(*pIter));
  2092   2084     pIter->pSeg = pSeg;
  2093         -  pIter->iIdx = iIdx;
  2094   2085   
  2095   2086     /* This block sets stack variable iPg to the leaf page number that may
  2096   2087     ** contain term (pTerm/nTerm), if it is present in the segment. */
  2097   2088     for(h=pSeg->nHeight-1; h>0; h--){
  2098   2089       Fts5NodeIter node;              /* For iterating through internal nodes */
  2099         -    i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, h, iPg);
         2090  +    i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
  2100   2091       Fts5Data *pNode = fts5DataRead(p, iRowid);
  2101   2092       if( pNode==0 ) break;
  2102   2093   
  2103   2094       fts5NodeIterInit(pNode->p, pNode->n, &node);
  2104   2095       assert( node.term.n==0 );
  2105   2096   
  2106   2097       iPg = node.iChild;
................................................................................
  2145   2136     if( p->rc==SQLITE_OK && bGe==0 ){
  2146   2137       pIter->flags |= FTS5_SEGITER_ONETERM;
  2147   2138       if( pIter->pLeaf ){
  2148   2139         if( flags & FTS5INDEX_QUERY_DESC ){
  2149   2140           pIter->flags |= FTS5_SEGITER_REVERSE;
  2150   2141         }
  2151   2142         if( bDlidx ){
  2152         -        fts5SegIterLoadDlidx(p, iIdx, pIter);
         2143  +        fts5SegIterLoadDlidx(p, pIter);
  2153   2144         }
  2154   2145         if( flags & FTS5INDEX_QUERY_DESC ){
  2155         -        fts5SegIterReverse(p, iIdx, pIter);
         2146  +        fts5SegIterReverse(p, pIter);
  2156   2147         }
  2157   2148       }
  2158   2149     }
  2159   2150   }
  2160   2151   
  2161   2152   /*
  2162   2153   ** Initialize the object pIter to point to term pTerm/nTerm within the
  2163         -** in-memory hash table iIdx. If there is no such term in the table, the 
         2154  +** in-memory hash table. If there is no such term in the hash-table, the 
  2164   2155   ** iterator is set to EOF.
  2165   2156   **
  2166   2157   ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
  2167   2158   ** an error has already occurred when this function is called, it is a no-op.
  2168   2159   */
  2169   2160   static void fts5SegIterHashInit(
  2170   2161     Fts5Index *p,                   /* FTS5 backend */
  2171         -  int iIdx,                       /* Config.aHash[] index of FTS index */
  2172   2162     const u8 *pTerm, int nTerm,     /* Term to seek to */
  2173   2163     int flags,                      /* Mask of FTS5INDEX_XXX flags */
  2174   2164     Fts5SegIter *pIter              /* Object to populate */
  2175   2165   ){
  2176         -  Fts5Hash *pHash = p->apHash[iIdx];
  2177   2166     const u8 *pList = 0;
  2178   2167     int nList = 0;
  2179   2168     const u8 *z = 0;
  2180   2169     int n = 0;
  2181   2170   
  2182         -  assert( pHash );
         2171  +  assert( p->pHash );
  2183   2172     assert( p->rc==SQLITE_OK );
  2184   2173   
  2185         -  if( pTerm==0 || (iIdx==0 && (flags & FTS5INDEX_QUERY_PREFIX)) ){
  2186         -    p->rc = sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm);
  2187         -    sqlite3Fts5HashScanEntry(pHash, (const char**)&z, &pList, &nList);
         2174  +  if( pTerm==0 || (flags & FTS5INDEX_QUERY_PREFIX) ){
         2175  +    p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm);
         2176  +    sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList);
  2188   2177       n = (z ? strlen((const char*)z) : 0);
  2189   2178     }else{
  2190   2179       pIter->flags |= FTS5_SEGITER_ONETERM;
  2191         -    sqlite3Fts5HashQuery(pHash, (const char*)pTerm, nTerm, &pList, &nList);
         2180  +    sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList);
  2192   2181       z = pTerm;
  2193   2182       n = nTerm;
  2194   2183     }
  2195   2184   
  2196   2185     if( pList ){
  2197   2186       Fts5Data *pLeaf;
  2198   2187       sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
................................................................................
  2548   2537   **
  2549   2538   ** The iterator initially points to the first term/rowid entry in the 
  2550   2539   ** iterated data.
  2551   2540   */
  2552   2541   static void fts5MultiIterNew(
  2553   2542     Fts5Index *p,                   /* FTS5 backend to iterate within */
  2554   2543     Fts5Structure *pStruct,         /* Structure of specific index */
  2555         -  int iIdx,                       /* Config.aHash[] index of FTS index */
  2556   2544     int bSkipEmpty,                 /* True to ignore delete-keys */
  2557   2545     int flags,                      /* FTS5INDEX_QUERY_XXX flags */
  2558   2546     const u8 *pTerm, int nTerm,     /* Term to seek to (or NULL/0) */
  2559   2547     int iLevel,                     /* Level to iterate (-1 for all) */
  2560   2548     int nSegment,                   /* Number of segments to merge (iLevel>=0) */
  2561   2549     Fts5MultiSegIter **ppOut        /* New object */
  2562   2550   ){
................................................................................
  2570   2558     assert( (pTerm==0 && nTerm==0) || iLevel<0 );
  2571   2559   
  2572   2560     /* Allocate space for the new multi-seg-iterator. */
  2573   2561     if( p->rc==SQLITE_OK ){
  2574   2562       if( iLevel<0 ){
  2575   2563         assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  2576   2564         nSeg = pStruct->nSegment;
  2577         -      nSeg += (p->apHash ? 1 : 0);
         2565  +      nSeg += (p->pHash ? 1 : 0);
  2578   2566       }else{
  2579   2567         nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
  2580   2568       }
  2581   2569       for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
  2582   2570     }
  2583   2571   
  2584   2572     *ppOut = pNew = fts5IdxMalloc(p, 
................................................................................
  2592   2580     pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot];
  2593   2581     pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC));
  2594   2582     pNew->bSkipEmpty = bSkipEmpty;
  2595   2583   
  2596   2584     /* Initialize each of the component segment iterators. */
  2597   2585     if( iLevel<0 ){
  2598   2586       Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
  2599         -    if( p->apHash ){
         2587  +    if( p->pHash ){
  2600   2588         /* Add a segment iterator for the current contents of the hash table. */
  2601   2589         Fts5SegIter *pIter = &pNew->aSeg[iIter++];
  2602         -      fts5SegIterHashInit(p, iIdx, pTerm, nTerm, flags, pIter);
         2590  +      fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
  2603   2591       }
  2604   2592       for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
  2605   2593         for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
  2606   2594           Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
  2607   2595           Fts5SegIter *pIter = &pNew->aSeg[iIter++];
  2608   2596           if( pTerm==0 ){
  2609         -          fts5SegIterInit(p, iIdx, pSeg, pIter);
         2597  +          fts5SegIterInit(p, pSeg, pIter);
  2610   2598           }else{
  2611         -          fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, flags, pSeg, pIter);
         2599  +          fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
  2612   2600           }
  2613   2601         }
  2614   2602       }
  2615   2603     }else{
  2616   2604       pLvl = &pStruct->aLevel[iLevel];
  2617   2605       for(iSeg=nSeg-1; iSeg>=0; iSeg--){
  2618         -      fts5SegIterInit(p, iIdx, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
         2606  +      fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]);
  2619   2607       }
  2620   2608     }
  2621   2609     assert( iIter==nSeg );
  2622   2610   
  2623   2611     /* If the above was successful, each component iterators now points 
  2624   2612     ** to the first entry in its segment. In this case initialize the 
  2625   2613     ** aFirst[] array. Or, if an error has occurred, free the iterator
................................................................................
  2730   2718     int iOff = pSeg->iLeafOffset;
  2731   2719   
  2732   2720     memset(pIter, 0, sizeof(*pIter));
  2733   2721     /* If Fts5SegIter.pSeg is NULL, then this iterator iterates through data
  2734   2722     ** currently stored in a hash table. In this case there is no leaf-rowid
  2735   2723     ** to calculate.  */
  2736   2724     if( pSeg->pSeg ){
  2737         -    int iId = pSeg->pSeg->iSegid;
  2738         -    i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno);
         2725  +    i64 rowid = FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, 0, pSeg->iLeafPgno);
  2739   2726       pIter->iLeafRowid = rowid;
  2740   2727     }
  2741   2728   
  2742   2729     fts5DataReference(pLeaf);
  2743   2730     pIter->pLeaf = pLeaf;
  2744   2731     pIter->nRem = pSeg->nPos;
  2745   2732     pIter->n = MIN(pLeaf->n - iOff, pIter->nRem);
................................................................................
  2790   2777     return (int)iSegid;
  2791   2778   }
  2792   2779   
  2793   2780   /*
  2794   2781   ** Discard all data currently cached in the hash-tables.
  2795   2782   */
  2796   2783   static void fts5IndexDiscardData(Fts5Index *p){
  2797         -  assert( p->apHash || p->nPendingData==0 );
  2798         -  if( p->apHash ){
  2799         -    Fts5Config *pConfig = p->pConfig;
  2800         -    int i;
  2801         -    for(i=0; i<=pConfig->nPrefix; i++){
  2802         -      sqlite3Fts5HashClear(p->apHash[i]);
  2803         -    }
         2784  +  assert( p->pHash || p->nPendingData==0 );
         2785  +  if( p->pHash ){
         2786  +    sqlite3Fts5HashClear(p->pHash);
  2804   2787       p->nPendingData = 0;
  2805   2788     }
  2806   2789   }
  2807   2790   
  2808   2791   /*
  2809   2792   ** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares
  2810   2793   ** with buffer (nOld/pOld).
................................................................................
  2828   2811   static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
  2829   2812     if( pWriter->nEmpty ){
  2830   2813       int bFlag = 0;
  2831   2814       Fts5PageWriter *pPg;
  2832   2815       pPg = &pWriter->aWriter[1];
  2833   2816       if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
  2834   2817         i64 iKey = FTS5_DOCLIST_IDX_ROWID(
  2835         -          pWriter->iIdx, pWriter->iSegid, 
  2836         -          pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty
         2818  +          pWriter->iSegid, pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty
  2837   2819         );
  2838   2820         assert( pWriter->cdlidx.n>0 );
  2839   2821         fts5DataWrite(p, iKey, pWriter->cdlidx.p, pWriter->cdlidx.n);
  2840   2822         bFlag = 1;
  2841   2823       }
  2842   2824       fts5BufferAppendVarint(&p->rc, &pPg->buf, bFlag);
  2843   2825       fts5BufferAppendVarint(&p->rc, &pPg->buf, pWriter->nEmpty);
................................................................................
  2897   2879       pPage = &pWriter->aWriter[iHeight];
  2898   2880   
  2899   2881       fts5WriteBtreeNEmpty(p, pWriter);
  2900   2882   
  2901   2883       if( pPage->buf.n>=p->pConfig->pgsz ){
  2902   2884         /* pPage will be written to disk. The term will be written into the
  2903   2885         ** parent of pPage.  */
  2904         -      i64 iRowid = FTS5_SEGMENT_ROWID(
  2905         -          pWriter->iIdx, pWriter->iSegid, iHeight, pPage->pgno
  2906         -      );
         2886  +      i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno);
  2907   2887         fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
  2908   2888         fts5BufferZero(&pPage->buf);
  2909   2889         fts5BufferZero(&pPage->term);
  2910   2890         fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno);
  2911   2891         pPage->pgno++;
  2912   2892       }else{
  2913   2893         int nPre = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm);
................................................................................
  2967   2947     if( pWriter->bFirstTermInPage ){
  2968   2948       /* No term was written to this page. */
  2969   2949       assert( 0==fts5GetU16(&pPage->buf.p[2]) );
  2970   2950       fts5WriteBtreeNoTerm(p, pWriter);
  2971   2951     }
  2972   2952   
  2973   2953     /* Write the current page to the db. */
  2974         -  iRowid = FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, 0, pPage->pgno);
         2954  +  iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, 0, pPage->pgno);
  2975   2955     fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n);
  2976   2956   
  2977   2957     /* Initialize the next page. */
  2978   2958     fts5BufferZero(&pPage->buf);
  2979   2959     fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
  2980   2960     pPage->pgno++;
  2981   2961   
................................................................................
  3175   3155           fts5WriteBtreeNEmpty(p, pWriter);
  3176   3156         }
  3177   3157         *pnHeight = pWriter->nWriter;
  3178   3158   
  3179   3159         for(i=1; i<pWriter->nWriter; i++){
  3180   3160           Fts5PageWriter *pPg = &pWriter->aWriter[i];
  3181   3161           fts5DataWrite(p, 
  3182         -            FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pPg->pgno), 
         3162  +            FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno), 
  3183   3163               pPg->buf.p, pPg->buf.n
  3184   3164           );
  3185   3165         }
  3186   3166       }
  3187   3167     }
  3188   3168     for(i=0; i<pWriter->nWriter; i++){
  3189   3169       Fts5PageWriter *pPg = &pWriter->aWriter[i];
................................................................................
  3193   3173     sqlite3_free(pWriter->aWriter);
  3194   3174     sqlite3Fts5BufferFree(&pWriter->cdlidx);
  3195   3175   }
  3196   3176   
  3197   3177   static void fts5WriteInit(
  3198   3178     Fts5Index *p, 
  3199   3179     Fts5SegWriter *pWriter, 
  3200         -  int iIdx, int iSegid
         3180  +  int iSegid
  3201   3181   ){
  3202   3182     memset(pWriter, 0, sizeof(Fts5SegWriter));
  3203         -  pWriter->iIdx = iIdx;
  3204   3183     pWriter->iSegid = iSegid;
  3205   3184   
  3206   3185     pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p,sizeof(Fts5PageWriter));
  3207   3186     if( pWriter->aWriter==0 ) return;
  3208   3187     pWriter->nWriter = 1;
  3209   3188     pWriter->aWriter[0].pgno = 1;
  3210   3189     pWriter->bFirstTermInPage = 1;
  3211   3190   }
  3212   3191   
  3213   3192   static void fts5WriteInitForAppend(
  3214   3193     Fts5Index *p,                   /* FTS5 backend object */
  3215   3194     Fts5SegWriter *pWriter,         /* Writer to initialize */
  3216         -  int iIdx,                       /* Index segment is a part of */
  3217   3195     Fts5StructureSegment *pSeg      /* Segment object to append to */
  3218   3196   ){
  3219   3197     int nByte = pSeg->nHeight * sizeof(Fts5PageWriter);
  3220   3198     memset(pWriter, 0, sizeof(Fts5SegWriter));
  3221         -  pWriter->iIdx = iIdx;
  3222   3199     pWriter->iSegid = pSeg->iSegid;
  3223   3200     pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte);
  3224   3201   
  3225   3202     if( p->rc==SQLITE_OK ){
  3226   3203       int pgno = 1;
  3227   3204       int i;
  3228   3205       pWriter->nWriter = pSeg->nHeight;
  3229   3206       pWriter->aWriter[0].pgno = pSeg->pgnoLast+1;
  3230   3207       for(i=pSeg->nHeight-1; i>0; i--){
  3231         -      i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pgno);
         3208  +      i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno);
  3232   3209         Fts5PageWriter *pPg = &pWriter->aWriter[i];
  3233   3210         pPg->pgno = pgno;
  3234   3211         fts5DataBuffer(p, &pPg->buf, iRowid);
  3235   3212         if( p->rc==SQLITE_OK ){
  3236   3213           Fts5NodeIter ss;
  3237   3214           fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss);
  3238   3215           while( ss.aData ) fts5NodeIterNext(&p->rc, &ss);
................................................................................
  3272   3249       }else{
  3273   3250         int iOff = pSeg->iTermLeafOffset;     /* Offset on new first leaf page */
  3274   3251         i64 iLeafRowid;
  3275   3252         Fts5Data *pData;
  3276   3253         int iId = pSeg->pSeg->iSegid;
  3277   3254         u8 aHdr[4] = {0x00, 0x00, 0x00, 0x04};
  3278   3255   
  3279         -      iLeafRowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iTermLeafPgno);
         3256  +      iLeafRowid = FTS5_SEGMENT_ROWID(iId, 0, pSeg->iTermLeafPgno);
  3280   3257         pData = fts5DataRead(p, iLeafRowid);
  3281   3258         if( pData ){
  3282   3259           fts5BufferZero(&buf);
  3283   3260           fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr);
  3284   3261           fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n);
  3285   3262           fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p);
  3286   3263           fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]);
  3287   3264           fts5DataRelease(pData);
  3288   3265           pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
  3289         -        fts5DataDelete(p, FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, 1),iLeafRowid);
         3266  +        fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid);
  3290   3267           fts5DataWrite(p, iLeafRowid, buf.p, buf.n);
  3291   3268         }
  3292   3269       }
  3293   3270     }
  3294   3271     fts5BufferFree(&buf);
  3295   3272   }
  3296   3273   
  3297   3274   /*
  3298   3275   **
  3299   3276   */
  3300   3277   static void fts5IndexMergeLevel(
  3301   3278     Fts5Index *p,                   /* FTS5 backend object */
  3302         -  int iIdx,                       /* Index to work on */
  3303         -  Fts5Structure **ppStruct,       /* IN/OUT: Stucture of index iIdx */
         3279  +  Fts5Structure **ppStruct,       /* IN/OUT: Stucture of index */
  3304   3280     int iLvl,                       /* Level to read input from */
  3305   3281     int *pnRem                      /* Write up to this many output leaves */
  3306   3282   ){
  3307   3283     Fts5Structure *pStruct = *ppStruct;
  3308   3284     Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl];
  3309   3285     Fts5StructureLevel *pLvlOut;
  3310   3286     Fts5MultiSegIter *pIter = 0;    /* Iterator to read input data */
................................................................................
  3317   3293     int bOldest;                    /* True if the output segment is the oldest */
  3318   3294   
  3319   3295     assert( iLvl<pStruct->nLevel );
  3320   3296     assert( pLvl->nMerge<=pLvl->nSeg );
  3321   3297   
  3322   3298     memset(&writer, 0, sizeof(Fts5SegWriter));
  3323   3299     memset(&term, 0, sizeof(Fts5Buffer));
  3324         -  writer.iIdx = iIdx;
  3325   3300     if( pLvl->nMerge ){
  3326   3301       pLvlOut = &pStruct->aLevel[iLvl+1];
  3327   3302       assert( pLvlOut->nSeg>0 );
  3328   3303       nInput = pLvl->nMerge;
  3329         -    fts5WriteInitForAppend(p, &writer, iIdx, &pLvlOut->aSeg[pLvlOut->nSeg-1]);
         3304  +    fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]);
  3330   3305       pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1];
  3331   3306     }else{
  3332   3307       int iSegid = fts5AllocateSegid(p, pStruct);
  3333   3308   
  3334   3309       /* Extend the Fts5Structure object as required to ensure the output
  3335   3310       ** segment exists. */
  3336   3311       if( iLvl==pStruct->nLevel-1 ){
................................................................................
  3338   3313         pStruct = *ppStruct;
  3339   3314       }
  3340   3315       fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0);
  3341   3316       if( p->rc ) return;
  3342   3317       pLvl = &pStruct->aLevel[iLvl];
  3343   3318       pLvlOut = &pStruct->aLevel[iLvl+1];
  3344   3319   
  3345         -    fts5WriteInit(p, &writer, iIdx, iSegid);
         3320  +    fts5WriteInit(p, &writer, iSegid);
  3346   3321   
  3347   3322       /* Add the new segment to the output level */
  3348   3323       pSeg = &pLvlOut->aSeg[pLvlOut->nSeg];
  3349   3324       pLvlOut->nSeg++;
  3350   3325       pSeg->pgnoFirst = 1;
  3351   3326       pSeg->iSegid = iSegid;
  3352   3327       pStruct->nSegment++;
................................................................................
  3358   3333   
  3359   3334   #if 0
  3360   3335   fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl);
  3361   3336   fflush(stdout);
  3362   3337   #endif
  3363   3338   
  3364   3339     assert( iLvl>=0 );
  3365         -  for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter);
         3340  +  for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter);
  3366   3341         fts5MultiIterEof(p, pIter)==0;
  3367   3342         fts5MultiIterNext(p, pIter, 0, 0)
  3368   3343     ){
  3369   3344       Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ];
  3370   3345       Fts5ChunkIter sPos;           /* Used to iterate through position list */
  3371   3346       int nPos;                     /* position-list size field value */
  3372   3347       int nTerm;
................................................................................
  3410   3385     fts5WriteFinish(p, &writer, &pSeg->nHeight, &pSeg->pgnoLast);
  3411   3386   
  3412   3387     if( fts5MultiIterEof(p, pIter) ){
  3413   3388       int i;
  3414   3389   
  3415   3390       /* Remove the redundant segments from the %_data table */
  3416   3391       for(i=0; i<nInput; i++){
  3417         -      fts5DataRemoveSegment(p, iIdx, pLvl->aSeg[i].iSegid);
         3392  +      fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid);
  3418   3393       }
  3419   3394   
  3420   3395       /* Remove the redundant segments from the input level */
  3421   3396       if( pLvl->nSeg!=nInput ){
  3422   3397         int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment);
  3423   3398         memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove);
  3424   3399       }
................................................................................
  3437   3412   
  3438   3413     fts5MultiIterFree(p, pIter);
  3439   3414     fts5BufferFree(&term);
  3440   3415     if( pnRem ) *pnRem -= writer.nLeafWritten;
  3441   3416   }
  3442   3417   
  3443   3418   /*
  3444         -** Do up to nPg pages of automerge work on index iIdx.
         3419  +** Do up to nPg pages of automerge work on the index.
  3445   3420   */
  3446   3421   static void fts5IndexMerge(
  3447   3422     Fts5Index *p,                   /* FTS5 backend object */
  3448         -  int iIdx,                       /* Index to work on */
  3449   3423     Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  3450   3424     int nPg                         /* Pages of work to do */
  3451   3425   ){
  3452   3426     int nRem = nPg;
  3453   3427     Fts5Structure *pStruct = *ppStruct;
  3454   3428     while( nRem>0 && p->rc==SQLITE_OK ){
  3455   3429       int iLvl;                   /* To iterate through levels */
................................................................................
  3481   3455   #endif
  3482   3456   
  3483   3457       if( nBest<p->pConfig->nAutomerge 
  3484   3458           && pStruct->aLevel[iBestLvl].nMerge==0 
  3485   3459         ){
  3486   3460         break;
  3487   3461       }
  3488         -    fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem);
         3462  +    fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem);
  3489   3463       if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){
  3490   3464         fts5StructurePromote(p, iBestLvl+1, pStruct);
  3491   3465       }
  3492   3466     }
  3493   3467     *ppStruct = pStruct;
  3494   3468   }
  3495   3469   
  3496   3470   /*
  3497   3471   ** A total of nLeaf leaf pages of data has just been flushed to a level-0
  3498         -** segments in index iIdx with structure pStruct. This function updates the
  3499         -** write-counter accordingly and, if necessary, performs incremental merge
  3500         -** work.
         3472  +** segment. This function updates the write-counter accordingly and, if
         3473  +** necessary, performs incremental merge work.
  3501   3474   **
  3502   3475   ** If an error occurs, set the Fts5Index.rc error code. If an error has 
  3503   3476   ** already occurred, this function is a no-op.
  3504   3477   */
  3505   3478   static void fts5IndexAutomerge(
  3506   3479     Fts5Index *p,                   /* FTS5 backend object */
  3507         -  int iIdx,                       /* Index to work on */
  3508   3480     Fts5Structure **ppStruct,       /* IN/OUT: Current structure of index */
  3509   3481     int nLeaf                       /* Number of output leaves just written */
  3510   3482   ){
  3511   3483     if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){
  3512   3484       Fts5Structure *pStruct = *ppStruct;
  3513   3485       i64 nWrite;                   /* Initial value of write-counter */
  3514   3486       int nWork;                    /* Number of work-quanta to perform */
................................................................................
  3516   3488   
  3517   3489       /* Update the write-counter. While doing so, set nWork. */
  3518   3490       nWrite = pStruct->nWriteCounter;
  3519   3491       nWork = ((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit);
  3520   3492       pStruct->nWriteCounter += nLeaf;
  3521   3493       nRem = p->nWorkUnit * nWork * pStruct->nLevel;
  3522   3494   
  3523         -    fts5IndexMerge(p, iIdx, ppStruct, nRem);
         3495  +    fts5IndexMerge(p, ppStruct, nRem);
  3524   3496     }
  3525   3497   }
  3526   3498   
  3527   3499   static void fts5IndexCrisismerge(
  3528   3500     Fts5Index *p,                   /* FTS5 backend object */
  3529         -  int iIdx,                       /* Index to work on */
  3530   3501     Fts5Structure **ppStruct        /* IN/OUT: Current structure of index */
  3531   3502   ){
  3532   3503     const int nCrisis = p->pConfig->nCrisisMerge;
  3533   3504     Fts5Structure *pStruct = *ppStruct;
  3534   3505     int iLvl = 0;
  3535   3506   
  3536   3507     assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 );
  3537   3508     while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
  3538         -    fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0);
         3509  +    fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
  3539   3510       fts5StructurePromote(p, iLvl+1, pStruct);
  3540   3511       iLvl++;
  3541   3512     }
  3542   3513     *ppStruct = pStruct;
  3543   3514   }
  3544   3515   
  3545   3516   static int fts5IndexReturn(Fts5Index *p){
................................................................................
  3580   3551   /*
  3581   3552   ** Flush the contents of in-memory hash table iHash to a new level-0 
  3582   3553   ** segment on disk. Also update the corresponding structure record.
  3583   3554   **
  3584   3555   ** If an error occurs, set the Fts5Index.rc error code. If an error has 
  3585   3556   ** already occurred, this function is a no-op.
  3586   3557   */
  3587         -static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
  3588         -  Fts5Hash *pHash = p->apHash[iHash];
         3558  +static void fts5FlushOneHash(Fts5Index *p){
         3559  +  Fts5Hash *pHash = p->pHash;
  3589   3560     Fts5Structure *pStruct;
  3590   3561     int iSegid;
  3591   3562     int pgnoLast = 0;                 /* Last leaf page number in segment */
  3592   3563   
  3593   3564     /* Obtain a reference to the index structure and allocate a new segment-id
  3594   3565     ** for the new level-0 segment.  */
  3595         -  pStruct = fts5StructureRead(p, iHash);
         3566  +  pStruct = fts5StructureRead(p);
  3596   3567     iSegid = fts5AllocateSegid(p, pStruct);
  3597   3568   
  3598   3569     if( iSegid ){
  3599   3570       const int pgsz = p->pConfig->pgsz;
  3600   3571   
  3601   3572       Fts5StructureSegment *pSeg;   /* New segment within pStruct */
  3602   3573       int nHeight;                  /* Height of new segment b-tree */
  3603   3574       Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
  3604   3575       const u8 *zPrev = 0;
  3605   3576   
  3606   3577       Fts5SegWriter writer;
  3607         -    fts5WriteInit(p, &writer, iHash, iSegid);
         3578  +    fts5WriteInit(p, &writer, iSegid);
  3608   3579   
  3609   3580       /* Pre-allocate the buffer used to assemble leaf pages to the target
  3610   3581       ** page size.  */
  3611   3582       assert( pgsz>0 );
  3612   3583       pBuf = &writer.aWriter[0].buf;
  3613   3584       fts5BufferGrow(&p->rc, pBuf, pgsz + 20);
  3614   3585   
................................................................................
  3745   3716         pSeg->pgnoLast = pgnoLast;
  3746   3717         pStruct->nSegment++;
  3747   3718       }
  3748   3719       fts5StructurePromote(p, 0, pStruct);
  3749   3720     }
  3750   3721   
  3751   3722   
  3752         -  fts5IndexAutomerge(p, iHash, &pStruct, pgnoLast);
  3753         -  fts5IndexCrisismerge(p, iHash, &pStruct);
  3754         -  fts5StructureWrite(p, iHash, pStruct);
         3723  +  fts5IndexAutomerge(p, &pStruct, pgnoLast);
         3724  +  fts5IndexCrisismerge(p, &pStruct);
         3725  +  fts5StructureWrite(p, pStruct);
  3755   3726     fts5StructureRelease(pStruct);
  3756   3727   }
  3757   3728   
  3758   3729   /*
  3759   3730   ** Flush any data stored in the in-memory hash tables to the database.
  3760   3731   */
  3761   3732   static void fts5IndexFlush(Fts5Index *p){
  3762         -  Fts5Config *pConfig = p->pConfig;
  3763         -  int i;                          /* Used to iterate through indexes */
  3764         -  int nLeaf = 0;                  /* Number of leaves written */
  3765         -
  3766         -  /* If an error has already occured this call is a no-op. */
  3767         -  if( p->nPendingData==0 ) return;
  3768         -  assert( p->apHash );
  3769         -
  3770         -  /* Flush the terms and each prefix index to disk */
  3771         -  for(i=0; i<=pConfig->nPrefix; i++){
  3772         -    fts5FlushOneHash(p, i, &nLeaf);
         3733  +  /* Unless it is empty, flush the hash table to disk */
         3734  +  if( p->rc==SQLITE_OK && p->nPendingData ){
         3735  +    assert( p->pHash );
         3736  +    p->nPendingData = 0;
         3737  +    fts5FlushOneHash(p);
  3773   3738     }
  3774         -  p->nPendingData = 0;
  3775   3739   }
  3776   3740   
  3777   3741   
  3778   3742   int sqlite3Fts5IndexOptimize(Fts5Index *p){
  3779         -  Fts5Config *pConfig = p->pConfig;
  3780         -  int i;
         3743  +  Fts5Structure *pStruct;
         3744  +  Fts5Structure *pNew = 0;
         3745  +  int nSeg = 0;
  3781   3746   
  3782   3747     assert( p->rc==SQLITE_OK );
  3783   3748     fts5IndexFlush(p);
  3784         -  for(i=0; i<=pConfig->nPrefix; i++){
  3785         -    Fts5Structure *pStruct = fts5StructureRead(p, i);
  3786         -    Fts5Structure *pNew = 0;
  3787         -    int nSeg = 0;
  3788         -    if( pStruct ){
  3789         -      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
  3790         -      nSeg = pStruct->nSegment;
  3791         -      if( nSeg>1 ){
  3792         -        int nByte = sizeof(Fts5Structure);
  3793         -        nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
  3794         -        pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
  3795         -      }
  3796         -    }
  3797         -    if( pNew ){
  3798         -      Fts5StructureLevel *pLvl;
  3799         -      int nByte = nSeg * sizeof(Fts5StructureSegment);
  3800         -      pNew->nLevel = pStruct->nLevel+1;
  3801         -      pNew->nWriteCounter = pStruct->nWriteCounter;
  3802         -      pLvl = &pNew->aLevel[pStruct->nLevel];
  3803         -      pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
  3804         -      if( pLvl->aSeg ){
  3805         -        int iLvl, iSeg;
  3806         -        int iSegOut = 0;
  3807         -        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  3808         -          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  3809         -            pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
  3810         -            iSegOut++;
  3811         -          }
         3749  +  pStruct = fts5StructureRead(p);
         3750  +
         3751  +  if( pStruct ){
         3752  +    assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
         3753  +    nSeg = pStruct->nSegment;
         3754  +    if( nSeg>1 ){
         3755  +      int nByte = sizeof(Fts5Structure);
         3756  +      nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
         3757  +      pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte);
         3758  +    }
         3759  +  }
         3760  +  if( pNew ){
         3761  +    Fts5StructureLevel *pLvl;
         3762  +    int nByte = nSeg * sizeof(Fts5StructureSegment);
         3763  +    pNew->nLevel = pStruct->nLevel+1;
         3764  +    pNew->nWriteCounter = pStruct->nWriteCounter;
         3765  +    pLvl = &pNew->aLevel[pStruct->nLevel];
         3766  +    pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
         3767  +    if( pLvl->aSeg ){
         3768  +      int iLvl, iSeg;
         3769  +      int iSegOut = 0;
         3770  +      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         3771  +        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         3772  +          pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg];
         3773  +          iSegOut++;
  3812   3774           }
  3813         -        pNew->nSegment = pLvl->nSeg = nSeg;
  3814         -      }else{
  3815         -        sqlite3_free(pNew);
  3816         -        pNew = 0;
  3817         -      }
  3818         -    }
  3819         -
  3820         -    if( pNew ){
  3821         -      int iLvl = pNew->nLevel-1;
  3822         -      while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
  3823         -        int nRem = FTS5_OPT_WORK_UNIT;
  3824         -        fts5IndexMergeLevel(p, i, &pNew, iLvl, &nRem);
  3825         -      }
  3826         -
  3827         -      fts5StructureWrite(p, i, pNew);
  3828         -      fts5StructureRelease(pNew);
  3829         -    }
  3830         -
  3831         -    fts5StructureRelease(pStruct);
  3832         -  }
  3833         -
         3775  +      }
         3776  +      pNew->nSegment = pLvl->nSeg = nSeg;
         3777  +    }else{
         3778  +      sqlite3_free(pNew);
         3779  +      pNew = 0;
         3780  +    }
         3781  +  }
         3782  +
         3783  +  if( pNew ){
         3784  +    int iLvl = pNew->nLevel-1;
         3785  +    while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){
         3786  +      int nRem = FTS5_OPT_WORK_UNIT;
         3787  +      fts5IndexMergeLevel(p, &pNew, iLvl, &nRem);
         3788  +    }
         3789  +
         3790  +    fts5StructureWrite(p, pNew);
         3791  +    fts5StructureRelease(pNew);
         3792  +  }
         3793  +
         3794  +  fts5StructureRelease(pStruct);
  3834   3795     return fts5IndexReturn(p); 
  3835   3796   }
  3836   3797   
  3837   3798   int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
  3838   3799     Fts5Structure *pStruct;
  3839   3800   
  3840         -  pStruct = fts5StructureRead(p, 0);
  3841         -  fts5IndexMerge(p, 0, &pStruct, nMerge);
  3842         -  fts5StructureWrite(p, 0, pStruct);
         3801  +  pStruct = fts5StructureRead(p);
         3802  +  fts5IndexMerge(p, &pStruct, nMerge);
         3803  +  fts5StructureWrite(p, pStruct);
  3843   3804     fts5StructureRelease(pStruct);
  3844   3805   
  3845   3806     return fts5IndexReturn(p);
  3846   3807   }
  3847   3808   
  3848   3809   
  3849   3810   /*
................................................................................
  4036   3997     Fts5IndexIter *pIter            /* Populate this object */
  4037   3998   ){
  4038   3999     Fts5Structure *pStruct;
  4039   4000     Fts5Buffer *aBuf;
  4040   4001     const int nBuf = 32;
  4041   4002   
  4042   4003     aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
  4043         -  pStruct = fts5StructureRead(p, 0);
         4004  +  pStruct = fts5StructureRead(p);
  4044   4005   
  4045   4006     if( aBuf && pStruct ){
  4046   4007       Fts5DoclistIter *pDoclist;
  4047   4008       int i;
  4048   4009       i64 iLastRowid = 0;
  4049   4010       Fts5MultiSegIter *p1 = 0;     /* Iterator used to gather data from index */
  4050   4011       Fts5Buffer doclist;
  4051   4012   
  4052   4013       memset(&doclist, 0, sizeof(doclist));
  4053         -    for(fts5MultiIterNew(p, pStruct, 0, 1, 1, pToken, nToken, -1, 0, &p1);
         4014  +    for(fts5MultiIterNew(p, pStruct, 1, 1, pToken, nToken, -1, 0, &p1);
  4054   4015           fts5MultiIterEof(p, p1)==0;
  4055   4016           fts5MultiIterNext(p, p1, 0, 0)
  4056   4017       ){
  4057   4018         i64 iRowid = fts5MultiIterRowid(p1);
  4058   4019         int nTerm;
  4059   4020         const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm);
  4060   4021         assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
................................................................................
  4109   4070   /*
  4110   4071   ** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain
  4111   4072   ** to the document with rowid iRowid.
  4112   4073   */
  4113   4074   int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){
  4114   4075     assert( p->rc==SQLITE_OK );
  4115   4076   
  4116         -  /* Allocate hash tables if they have not already been allocated */
  4117         -  if( p->apHash==0 ){
  4118         -    int i;
  4119         -    int rc = SQLITE_OK;
  4120         -    int nHash = p->pConfig->nPrefix + 1;
  4121         -    Fts5Hash **apNew;
  4122         -
  4123         -    apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash);
  4124         -    for(i=0; rc==SQLITE_OK && i<nHash; i++){
  4125         -      rc = sqlite3Fts5HashNew(&apNew[i], &p->nPendingData);
  4126         -    }
  4127         -    if( rc==SQLITE_OK ){
  4128         -      p->apHash = apNew;
  4129         -    }else{
  4130         -      if( apNew ){
  4131         -        for(i=0; i<nHash; i++){
  4132         -          sqlite3Fts5HashFree(apNew[i]);
  4133         -        }
  4134         -        sqlite3_free(apNew);
  4135         -      }
  4136         -      return rc;
  4137         -    }
         4077  +  /* Allocate the hash table if it has not already been allocated */
         4078  +  if( p->pHash==0 ){
         4079  +    p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData);
  4138   4080     }
  4139   4081   
  4140   4082     if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){
  4141         -    assert( p->rc==SQLITE_OK );
  4142   4083       fts5IndexFlush(p);
  4143   4084     }
  4144   4085     p->iWriteRowid = iRowid;
  4145   4086     return fts5IndexReturn(p);
  4146   4087   }
  4147   4088   
  4148   4089   /*
................................................................................
  4170   4111   
  4171   4112   /*
  4172   4113   ** The %_data table is completely empty when this function is called. This
  4173   4114   ** function populates it with the initial structure objects for each index,
  4174   4115   ** and the initial version of the "averages" record (a zero-byte blob).
  4175   4116   */
  4176   4117   int sqlite3Fts5IndexReinit(Fts5Index *p){
  4177         -  int i;
  4178   4118     Fts5Structure s;
  4179   4119   
         4120  +  assert( p->rc==SQLITE_OK );
         4121  +  p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
         4122  +
  4180   4123     memset(&s, 0, sizeof(Fts5Structure));
  4181         -  for(i=0; i<p->pConfig->nPrefix+1; i++){
  4182         -    fts5StructureWrite(p, i, &s);
  4183         -  }
  4184         -  if( p->rc==SQLITE_OK ){
  4185         -    p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
  4186         -  }
         4124  +  fts5StructureWrite(p, &s);
  4187   4125   
  4188   4126     return fts5IndexReturn(p);
  4189   4127   }
  4190   4128   
  4191   4129   /*
  4192   4130   ** Open a new Fts5Index handle. If the bCreate argument is true, create
  4193   4131   ** and initialize the underlying %_data table.
................................................................................
  4235   4173   */
  4236   4174   int sqlite3Fts5IndexClose(Fts5Index *p){
  4237   4175     int rc = SQLITE_OK;
  4238   4176     if( p ){
  4239   4177       assert( p->pReader==0 );
  4240   4178       sqlite3_finalize(p->pWriter);
  4241   4179       sqlite3_finalize(p->pDeleter);
  4242         -    if( p->apHash ){
  4243         -      int i;
  4244         -      for(i=0; i<=p->pConfig->nPrefix; i++){
  4245         -        sqlite3Fts5HashFree(p->apHash[i]);
  4246         -      }
  4247         -      sqlite3_free(p->apHash);
  4248         -    }
         4180  +    sqlite3Fts5HashFree(p->pHash);
         4181  +    sqlite3Fts5BufferFree(&p->scratch);
  4249   4182       sqlite3_free(p->zDataTbl);
  4250   4183       sqlite3_free(p);
  4251   4184     }
  4252   4185     return rc;
  4253   4186   }
  4254   4187   
  4255   4188   /*
................................................................................
  4298   4231   int sqlite3Fts5IndexWrite(
  4299   4232     Fts5Index *p,                   /* Index to write to */
  4300   4233     int iCol,                       /* Column token appears in (-ve -> delete) */
  4301   4234     int iPos,                       /* Position of token within column */
  4302   4235     const char *pToken, int nToken  /* Token to add or remove to or from index */
  4303   4236   ){
  4304   4237     int i;                          /* Used to iterate through indexes */
  4305         -  int rc;                         /* Return code */
         4238  +  int rc = SQLITE_OK;             /* Return code */
  4306   4239     Fts5Config *pConfig = p->pConfig;
  4307   4240   
  4308   4241     assert( p->rc==SQLITE_OK );
  4309   4242   
  4310         -  /* Add the new token to the main terms hash table. And to each of the
  4311         -  ** prefix hash tables that it is large enough for. */
         4243  +  /* Add the entry to the main terms index. */
  4312   4244     rc = sqlite3Fts5HashWrite(
  4313         -      p->apHash[0], p->iWriteRowid, iCol, iPos, pToken, nToken
         4245  +      p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken
  4314   4246     );
         4247  +
  4315   4248     for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){
  4316   4249       int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]);
  4317   4250       if( nByte ){
  4318         -      rc = sqlite3Fts5HashWrite(
  4319         -          p->apHash[i+1], p->iWriteRowid, iCol, iPos, pToken, nByte
         4251  +      rc = sqlite3Fts5HashWrite(p->pHash, 
         4252  +          p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX+i+1, pToken, nByte
  4320   4253         );
  4321   4254       }
  4322   4255     }
  4323   4256   
  4324   4257     return rc;
  4325   4258   }
  4326   4259   
................................................................................
  4333   4266     const char *pToken, int nToken, /* Token (or prefix) to query for */
  4334   4267     int flags,                      /* Mask of FTS5INDEX_QUERY_X flags */
  4335   4268     Fts5IndexIter **ppIter          /* OUT: New iterator object */
  4336   4269   ){
  4337   4270     Fts5Config *pConfig = p->pConfig;
  4338   4271     Fts5IndexIter *pRet;
  4339   4272     int iIdx = 0;
         4273  +  Fts5Buffer buf = {0, 0, 0};
         4274  +
         4275  +  if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){
         4276  +    memcpy(&buf.p[1], pToken, nToken);
         4277  +  }
  4340   4278   
  4341   4279     if( flags & FTS5INDEX_QUERY_PREFIX ){
  4342   4280       if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){
  4343   4281         iIdx = 1+pConfig->nPrefix;
  4344   4282       }else{
  4345   4283         int nChar = fts5IndexCharlen(pToken, nToken);
  4346   4284         for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){
................................................................................
  4351   4289   
  4352   4290     pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter));
  4353   4291     if( pRet ){
  4354   4292       memset(pRet, 0, sizeof(Fts5IndexIter));
  4355   4293   
  4356   4294       pRet->pIndex = p;
  4357   4295       if( iIdx<=pConfig->nPrefix ){
  4358         -      pRet->pStruct = fts5StructureRead(p, iIdx);
         4296  +      buf.p[0] = FTS5_MAIN_PREFIX + iIdx;
         4297  +      pRet->pStruct = fts5StructureRead(p);
  4359   4298         if( pRet->pStruct ){
  4360         -        fts5MultiIterNew(p, pRet->pStruct, 
  4361         -            iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
         4299  +        int f = (flags & ~FTS5INDEX_QUERY_PREFIX);
         4300  +        fts5MultiIterNew(
         4301  +            p, pRet->pStruct, 1, f, buf.p, nToken+1, -1, 0, &pRet->pMulti
  4362   4302           );
  4363   4303         }
  4364   4304       }else{
  4365   4305         int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
  4366         -      fts5SetupPrefixIter(p, bDesc, (const u8*)pToken, nToken, pRet);
         4306  +      buf.p[0] = FTS5_MAIN_PREFIX;
         4307  +      fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pRet);
  4367   4308       }
  4368   4309     }
  4369   4310   
  4370   4311     if( p->rc ){
  4371   4312       sqlite3Fts5IterClose(pRet);
  4372   4313       pRet = 0;
  4373   4314     }
  4374   4315     *ppIter = pRet;
         4316  +  sqlite3Fts5BufferFree(&buf);
  4375   4317     return fts5IndexReturn(p);
  4376   4318   }
  4377   4319   
  4378   4320   /*
  4379   4321   ** Return true if the iterator passed as the only argument is at EOF.
  4380   4322   */
  4381   4323   int sqlite3Fts5IterEof(Fts5IndexIter *pIter){
................................................................................
  4516   4458   ** Set the 32-bit cookie value stored at the start of all structure 
  4517   4459   ** records to the value passed as the second argument.
  4518   4460   **
  4519   4461   ** Return SQLITE_OK if successful, or an SQLite error code if an error
  4520   4462   ** occurs.
  4521   4463   */
  4522   4464   int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){
  4523         -  int rc = SQLITE_OK;
  4524         -  Fts5Config *pConfig = p->pConfig;
  4525         -  u8 aCookie[4];
  4526         -  int i;
         4465  +  int rc;                              /* Return code */
         4466  +  Fts5Config *pConfig = p->pConfig;    /* Configuration object */
         4467  +  u8 aCookie[4];                       /* Binary representation of iNew */
  4527   4468   
  4528   4469     assert( p->rc==SQLITE_OK );
         4470  +
  4529   4471     sqlite3Fts5Put32(aCookie, iNew);
  4530         -  for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){
  4531         -    sqlite3_blob *pBlob = 0;
  4532         -    i64 iRowid = FTS5_STRUCTURE_ROWID(i);
  4533         -    rc = sqlite3_blob_open(
  4534         -        pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob
  4535         -    );
  4536         -    if( rc==SQLITE_OK ){
  4537         -      sqlite3_blob_write(pBlob, aCookie, 4, 0);
  4538         -      rc = sqlite3_blob_close(pBlob);
  4539         -    }
         4472  +  sqlite3_blob *pBlob = 0;
         4473  +  rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, 
         4474  +      "block", FTS5_STRUCTURE_ROWID, 1, &pBlob
         4475  +  );
         4476  +  if( rc==SQLITE_OK ){
         4477  +    sqlite3_blob_write(pBlob, aCookie, 4, 0);
         4478  +    rc = sqlite3_blob_close(pBlob);
  4540   4479     }
  4541   4480   
  4542   4481     return rc;
  4543   4482   }
  4544   4483   
  4545   4484   int sqlite3Fts5IndexLoadConfig(Fts5Index *p){
  4546   4485     Fts5Structure *pStruct;
  4547         -  pStruct = fts5StructureRead(p, 0);
         4486  +  pStruct = fts5StructureRead(p);
  4548   4487     fts5StructureRelease(pStruct);
  4549   4488     return fts5IndexReturn(p);
  4550   4489   }
  4551   4490   
  4552   4491   
  4553   4492   /*************************************************************************
  4554   4493   **************************************************************************
................................................................................
  4559   4498   /*
  4560   4499   ** Return a simple checksum value based on the arguments.
  4561   4500   */
  4562   4501   static u64 fts5IndexEntryCksum(
  4563   4502     i64 iRowid, 
  4564   4503     int iCol, 
  4565   4504     int iPos, 
  4566         -  const char *pTerm, 
         4505  +  int iIdx,
         4506  +  const char *pTerm,
  4567   4507     int nTerm
  4568   4508   ){
  4569   4509     int i;
  4570   4510     u64 ret = iRowid;
  4571   4511     ret += (ret<<3) + iCol;
  4572   4512     ret += (ret<<3) + iPos;
         4513  +  if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx);
  4573   4514     for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i];
  4574   4515     return ret;
  4575   4516   }
  4576   4517   
  4577   4518   static void fts5BtreeIterInit(
  4578   4519     Fts5Index *p, 
  4579         -  int iIdx,
  4580   4520     Fts5StructureSegment *pSeg, 
  4581   4521     Fts5BtreeIter *pIter
  4582   4522   ){
  4583   4523     int nByte;
  4584   4524     int i;
  4585   4525     nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1);
  4586   4526     memset(pIter, 0, sizeof(*pIter));
  4587   4527     if( nByte ){
  4588   4528       pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte);
  4589   4529     }
  4590   4530     if( p->rc==SQLITE_OK ){
  4591   4531       pIter->nLvl = pSeg->nHeight-1;
  4592         -    pIter->iIdx = iIdx;
  4593   4532       pIter->p = p;
  4594   4533       pIter->pSeg = pSeg;
  4595   4534     }
  4596   4535     for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){
  4597         -    i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, i+1, 1);
         4536  +    i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1);
  4598   4537       Fts5Data *pData;
  4599   4538       pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid);
  4600   4539       if( pData ){
  4601   4540         fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s);
  4602   4541       }
  4603   4542     }
  4604   4543   
................................................................................
  4631   4570     }
  4632   4571     if( i==pIter->nLvl || p->rc ){
  4633   4572       pIter->bEof = 1;
  4634   4573     }else{
  4635   4574       int iSegid = pIter->pSeg->iSegid;
  4636   4575       for(i--; i>=0; i--){
  4637   4576         Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i];
  4638         -      i64 iRowid = FTS5_SEGMENT_ROWID(pIter->iIdx,iSegid,i+1,pLvl[1].s.iChild);
         4577  +      i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild);
  4639   4578         pLvl->pData = fts5DataRead(p, iRowid);
  4640   4579         if( pLvl->pData ){
  4641   4580           fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s);
  4642   4581         }
  4643   4582       }
  4644   4583     }
  4645   4584   
................................................................................
  4664   4603   
  4665   4604   /*
  4666   4605   ** This function is purely an internal test. It does not contribute to 
  4667   4606   ** FTS functionality, or even the integrity-check, in any way.
  4668   4607   **
  4669   4608   ** Instead, it tests that the same set of pgno/rowid combinations are 
  4670   4609   ** visited regardless of whether the doclist-index identified by parameters
  4671         -** iIdx/iSegid/iLeaf is iterated in forwards or reverse order.
         4610  +** iSegid/iLeaf is iterated in forwards or reverse order.
  4672   4611   */
  4673   4612   #ifdef SQLITE_DEBUG
  4674   4613   static void fts5DlidxIterTestReverse(
  4675   4614     Fts5Index *p, 
  4676         -  int iIdx,                       /* Index to load doclist-index from */
  4677   4615     int iSegid,                     /* Segment id to load from */
  4678   4616     int iLeaf                       /* Load doclist-index for this leaf */
  4679   4617   ){
  4680   4618     Fts5DlidxIter *pDlidx = 0;
  4681   4619     i64 cksum1 = 13;
  4682   4620     i64 cksum2 = 13;
  4683   4621   
  4684         -  for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iLeaf);
         4622  +  for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf);
  4685   4623         fts5DlidxIterEof(p, pDlidx)==0;
  4686   4624         fts5DlidxIterNext(pDlidx)
  4687   4625     ){
  4688   4626       assert( pDlidx->iLeafPgno>iLeaf );
  4689   4627       cksum1 = (cksum1 ^ ( (i64)(pDlidx->iLeafPgno) << 32 ));
  4690   4628       cksum1 = (cksum1 ^ pDlidx->iRowid);
  4691   4629     }
  4692   4630     fts5DlidxIterFree(pDlidx);
  4693   4631     pDlidx = 0;
  4694   4632   
  4695         -  for(pDlidx=fts5DlidxIterInit(p, 1, iIdx, iSegid, iLeaf);
         4633  +  for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf);
  4696   4634         fts5DlidxIterEof(p, pDlidx)==0;
  4697   4635         fts5DlidxIterPrev(pDlidx)
  4698   4636     ){
  4699   4637       assert( pDlidx->iLeafPgno>iLeaf );
  4700   4638       cksum2 = (cksum2 ^ ( (i64)(pDlidx->iLeafPgno) << 32 ));
  4701   4639       cksum2 = (cksum2 ^ pDlidx->iRowid);
  4702   4640     }
  4703   4641     fts5DlidxIterFree(pDlidx);
  4704   4642     pDlidx = 0;
  4705   4643   
  4706   4644     if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; 
  4707   4645   }
  4708   4646   #else
  4709         -# define fts5DlidxIterTestReverse(w,x,y,z)
         4647  +# define fts5DlidxIterTestReverse(x,y,z)
  4710   4648   #endif
  4711   4649   
  4712   4650   static void fts5IndexIntegrityCheckSegment(
  4713   4651     Fts5Index *p,                   /* FTS5 backend object */
  4714         -  int iIdx,                       /* Index that pSeg is a part of */
  4715   4652     Fts5StructureSegment *pSeg      /* Segment to check internal consistency */
  4716   4653   ){
  4717   4654     Fts5BtreeIter iter;             /* Used to iterate through b-tree hierarchy */
  4718   4655   
  4719   4656     if( pSeg->pgnoFirst==0 ) return;
  4720   4657   
  4721   4658     /* Iterate through the b-tree hierarchy.  */
  4722         -  for(fts5BtreeIterInit(p, iIdx, pSeg, &iter);
         4659  +  for(fts5BtreeIterInit(p, pSeg, &iter);
  4723   4660         p->rc==SQLITE_OK && iter.bEof==0;
  4724   4661         fts5BtreeIterNext(&iter)
  4725   4662     ){
  4726   4663       i64 iRow;                     /* Rowid for this leaf */
  4727   4664       Fts5Data *pLeaf;              /* Data for this leaf */
  4728   4665       int iOff;                     /* Offset of first term on leaf */
  4729   4666       int i;                        /* Used to iterate through empty leaves */
  4730   4667   
  4731   4668       /* If the leaf in question has already been trimmed from the segment, 
  4732   4669       ** ignore this b-tree entry. Otherwise, load it into memory. */
  4733   4670       if( iter.iLeaf<pSeg->pgnoFirst ) continue;
  4734         -    iRow = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, iter.iLeaf);
         4671  +    iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf);
  4735   4672       pLeaf = fts5DataRead(p, iRow);
  4736   4673       if( pLeaf==0 ) break;
  4737   4674   
  4738   4675       /* Check that the leaf contains at least one term, and that it is equal
  4739   4676       ** to or larger than the split-key in iter.term.  */
  4740   4677       iOff = fts5GetU16(&pLeaf->p[2]);
  4741   4678       if( iOff==0 ){
................................................................................
  4767   4704       if( iter.bDlidx ){
  4768   4705         Fts5DlidxIter *pDlidx = 0;  /* For iterating through doclist index */
  4769   4706         int iPrevLeaf = iter.iLeaf;
  4770   4707         int iSegid = pSeg->iSegid;
  4771   4708         int iPg;
  4772   4709         i64 iKey;
  4773   4710   
  4774         -      for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iter.iLeaf);
         4711  +      for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf);
  4775   4712             fts5DlidxIterEof(p, pDlidx)==0;
  4776   4713             fts5DlidxIterNext(pDlidx)
  4777   4714         ){
  4778   4715   
  4779   4716           /* Check any rowid-less pages that occur before the current leaf. */
  4780   4717           for(iPg=iPrevLeaf+1; iPg<pDlidx->iLeafPgno; iPg++){
  4781         -          iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg);
         4718  +          iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg);
  4782   4719             pLeaf = fts5DataRead(p, iKey);
  4783   4720             if( pLeaf ){
  4784   4721               if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
  4785   4722               fts5DataRelease(pLeaf);
  4786   4723             }
  4787   4724           }
  4788   4725           iPrevLeaf = pDlidx->iLeafPgno;
  4789   4726   
  4790   4727           /* Check that the leaf page indicated by the iterator really does
  4791   4728           ** contain the rowid suggested by the same. */
  4792         -        iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pDlidx->iLeafPgno);
         4729  +        iKey = FTS5_SEGMENT_ROWID(iSegid, 0, pDlidx->iLeafPgno);
  4793   4730           pLeaf = fts5DataRead(p, iKey);
  4794   4731           if( pLeaf ){
  4795   4732             i64 iRowid;
  4796   4733             int iRowidOff = fts5GetU16(&pLeaf->p[0]);
  4797   4734             getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
  4798   4735             if( iRowid!=pDlidx->iRowid ) p->rc = FTS5_CORRUPT;
  4799   4736             fts5DataRelease(pLeaf);
  4800   4737           }
  4801   4738   
  4802   4739         }
  4803   4740   
  4804   4741         for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){
  4805         -        iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg);
         4742  +        iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg);
  4806   4743           pLeaf = fts5DataRead(p, iKey);
  4807   4744           if( pLeaf ){
  4808   4745             if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT;
  4809   4746             fts5DataRelease(pLeaf);
  4810   4747           }
  4811   4748         }
  4812   4749   
  4813   4750         fts5DlidxIterFree(pDlidx);
  4814         -      fts5DlidxIterTestReverse(p, iIdx, iSegid, iter.iLeaf);
         4751  +      fts5DlidxIterTestReverse(p, iSegid, iter.iLeaf);
  4815   4752       }
  4816   4753     }
  4817   4754   
  4818   4755     /* Either iter.iLeaf must be the rightmost leaf-page in the segment, or 
  4819   4756     ** else the segment has been completely emptied by an ongoing merge
  4820   4757     ** operation. */
  4821   4758     if( p->rc==SQLITE_OK 
................................................................................
  4826   4763     }
  4827   4764   
  4828   4765     fts5BtreeIterFree(&iter);
  4829   4766   }
  4830   4767   
  4831   4768   
  4832   4769   static int fts5QueryCksum(
  4833         -  Fts5Index *p,
  4834         -  const char *z,
  4835         -  int n,
  4836         -  int flags,
  4837         -  u64 *pCksum
         4770  +  Fts5Index *p,                   /* Fts5 index object */
         4771  +  int iIdx,
         4772  +  const char *z,                  /* Index key to query for */
         4773  +  int n,                          /* Size of index key in bytes */
         4774  +  int flags,                      /* Flags for Fts5IndexQuery */
         4775  +  u64 *pCksum                     /* IN/OUT: Checksum value */
  4838   4776   ){
  4839   4777     u64 cksum = *pCksum;
  4840   4778     Fts5IndexIter *pIdxIter = 0;
  4841   4779     int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter);
  4842   4780   
  4843   4781     while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){
  4844   4782       const u8 *pPos;
................................................................................
  4849   4787         Fts5PoslistReader sReader;
  4850   4788         for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader);
  4851   4789             sReader.bEof==0;
  4852   4790             sqlite3Fts5PoslistReaderNext(&sReader)
  4853   4791         ){
  4854   4792           int iCol = FTS5_POS2COLUMN(sReader.iPos);
  4855   4793           int iOff = FTS5_POS2OFFSET(sReader.iPos);
  4856         -        cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n);
         4794  +        cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n);
  4857   4795         }
  4858   4796         rc = sqlite3Fts5IterNext(pIdxIter);
  4859   4797       }
  4860   4798     }
  4861   4799     sqlite3Fts5IterClose(pIdxIter);
  4862   4800   
  4863   4801     *pCksum = cksum;
................................................................................
  4871   4809   **
  4872   4810   ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the
  4873   4811   ** checksum does not match. Return SQLITE_OK if all checks pass without
  4874   4812   ** error, or some other SQLite error code if another error (e.g. OOM)
  4875   4813   ** occurs.
  4876   4814   */
  4877   4815   int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
  4878         -  Fts5Config *pConfig = p->pConfig;
  4879         -  int iIdx;                       /* Used to iterate through indexes */
  4880   4816     u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
  4881   4817     u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
  4882   4818     Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
  4883   4819     Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
         4820  +  Fts5MultiSegIter *pIter;        /* Used to iterate through entire index */
         4821  +  Fts5Structure *pStruct;         /* Index structure */
         4822  +  
         4823  +  /* Load the FTS index structure */
         4824  +  pStruct = fts5StructureRead(p);
  4884   4825   
  4885   4826     /* Check that the internal nodes of each segment match the leaves */
  4886         -  for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
  4887         -    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
  4888         -    if( pStruct ){
  4889         -      int iLvl, iSeg;
  4890         -      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
  4891         -        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
  4892         -          Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
  4893         -          fts5IndexIntegrityCheckSegment(p, iIdx, pSeg);
  4894         -        }
         4827  +  if( pStruct ){
         4828  +    int iLvl, iSeg;
         4829  +    for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
         4830  +      for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
         4831  +        Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg];
         4832  +        fts5IndexIntegrityCheckSegment(p, pSeg);
  4895   4833         }
  4896   4834       }
  4897         -    fts5StructureRelease(pStruct);
  4898   4835     }
  4899   4836   
  4900   4837     /* The cksum argument passed to this function is a checksum calculated
  4901   4838     ** based on all expected entries in the FTS index (including prefix index
  4902   4839     ** entries). This block checks that a checksum calculated based on the
  4903   4840     ** actual contents of FTS index is identical.
  4904   4841     **
................................................................................
  4906   4843     ** variable cksum2) based on entries extracted from the full-text index
  4907   4844     ** while doing a linear scan of each individual index in turn. 
  4908   4845     **
  4909   4846     ** As each term visited by the linear scans, a separate query for the
  4910   4847     ** same term is performed. cksum3 is calculated based on the entries
  4911   4848     ** extracted by these queries.
  4912   4849     */
  4913         -  for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){
  4914         -    Fts5MultiSegIter *pIter;
  4915         -    Fts5Structure *pStruct = fts5StructureRead(p, iIdx);
  4916         -    for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter);
  4917         -        fts5MultiIterEof(p, pIter)==0;
  4918         -        fts5MultiIterNext(p, pIter, 0, 0)
  4919         -    ){
  4920         -      int n;                      /* Size of term in bytes */
  4921         -      i64 iPos = 0;               /* Position read from poslist */
  4922         -      int iOff = 0;               /* Offset within poslist */
  4923         -      i64 iRowid = fts5MultiIterRowid(pIter);
  4924         -      char *z = (char*)fts5MultiIterTerm(pIter, &n);
  4925         -
  4926         -      poslist.n = 0;
  4927         -      fts5MultiIterPoslist(p, pIter, 0, &poslist);
  4928         -      while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
  4929         -        int iCol = FTS5_POS2COLUMN(iPos);
  4930         -        int iTokOff = FTS5_POS2OFFSET(iPos);
  4931         -        cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n);
         4850  +  for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter);
         4851  +      fts5MultiIterEof(p, pIter)==0;
         4852  +      fts5MultiIterNext(p, pIter, 0, 0)
         4853  +  ){
         4854  +    int n;                      /* Size of term in bytes */
         4855  +    i64 iPos = 0;               /* Position read from poslist */
         4856  +    int iOff = 0;               /* Offset within poslist */
         4857  +    i64 iRowid = fts5MultiIterRowid(pIter);
         4858  +    char *z = (char*)fts5MultiIterTerm(pIter, &n);
         4859  +
         4860  +    poslist.n = 0;
         4861  +    fts5MultiIterPoslist(p, pIter, 0, &poslist);
         4862  +    while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
         4863  +      int iCol = FTS5_POS2COLUMN(iPos);
         4864  +      int iTokOff = FTS5_POS2OFFSET(iPos);
         4865  +      cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n);
         4866  +    }
         4867  +
         4868  +    /* If this is a new term, query for it. Update cksum3 with the results. */
         4869  +    if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){
         4870  +      const char *zTerm = &z[1];     /* The term without the prefix-byte */
         4871  +      int nTerm = n-1;               /* Size of zTerm in bytes */
         4872  +      int iIdx = (z[0] - FTS5_MAIN_PREFIX);
         4873  +      int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
         4874  +      int rc;
         4875  +      u64 ck1 = 0;
         4876  +      u64 ck2 = 0;
         4877  +
         4878  +      /* Check that the results returned for ASC and DESC queries are
         4879  +      ** the same. If not, call this corruption.  */
         4880  +      rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1);
         4881  +      if( rc==SQLITE_OK ){
         4882  +        int f = flags|FTS5INDEX_QUERY_DESC;
         4883  +        rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
         4884  +      }
         4885  +      if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         4886  +
         4887  +      /* If this is a prefix query, check that the results returned if the
         4888  +      ** the index is disabled are the same. In both ASC and DESC order. */
         4889  +      if( iIdx>0 && rc==SQLITE_OK ){
         4890  +        int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
         4891  +        ck2 = 0;
         4892  +        rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
         4893  +        if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
         4894  +      }
         4895  +      if( iIdx>0 && rc==SQLITE_OK ){
         4896  +        int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
         4897  +        ck2 = 0;
         4898  +        rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2);
         4899  +        if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  4932   4900         }
  4933   4901   
  4934         -      /* If this is a new term, query for it. Update cksum3 with the results. */
  4935         -      if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){
  4936         -        int rc;
  4937         -        int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX);
  4938         -        u64 ck1 = 0;
  4939         -        u64 ck2 = 0;
  4940         -
  4941         -        /* Check that the results returned for ASC and DESC queries are
  4942         -        ** the same. If not, call this corruption.  */
  4943         -        rc = fts5QueryCksum(p, z, n, flags, &ck1);
  4944         -        if( rc==SQLITE_OK ){
  4945         -          rc = fts5QueryCksum(p, z, n, flags|FTS5INDEX_QUERY_DESC, &ck2);
  4946         -        }
  4947         -        if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  4948         -
  4949         -        /* If this is a prefix query, check that the results returned if the
  4950         -        ** the index is disabled are the same. In both ASC and DESC order. */
  4951         -        if( iIdx>0 && rc==SQLITE_OK ){
  4952         -          int f = flags|FTS5INDEX_QUERY_TEST_NOIDX;
  4953         -          ck2 = 0;
  4954         -          rc = fts5QueryCksum(p, z, n, f, &ck2);
  4955         -          if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  4956         -        }
  4957         -        if( iIdx>0 && rc==SQLITE_OK ){
  4958         -          int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC;
  4959         -          ck2 = 0;
  4960         -          rc = fts5QueryCksum(p, z, n, f, &ck2);
  4961         -          if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT;
  4962         -        }
  4963         -
  4964         -        cksum3 ^= ck1;
  4965         -        fts5BufferSet(&rc, &term, n, (const u8*)z);
  4966         -        p->rc = rc;
  4967         -      }
         4902  +      cksum3 ^= ck1;
         4903  +      fts5BufferSet(&rc, &term, n, (const u8*)z);
         4904  +      p->rc = rc;
  4968   4905       }
  4969         -    fts5MultiIterFree(p, pIter);
  4970         -    fts5StructureRelease(pStruct);
  4971   4906     }
         4907  +  fts5MultiIterFree(p, pIter);
         4908  +
  4972   4909     if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
  4973   4910     if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT;
  4974   4911   
         4912  +  fts5StructureRelease(pStruct);
  4975   4913     fts5BufferFree(&term);
  4976   4914     fts5BufferFree(&poslist);
  4977   4915     return fts5IndexReturn(p);
  4978   4916   }
  4979   4917   
  4980   4918   
  4981   4919   /*
................................................................................
  4989   4927     int iCol,                       /* Column term appears in */
  4990   4928     int iPos,                       /* Position term appears in */
  4991   4929     const char *pTerm, int nTerm    /* Term at iPos */
  4992   4930   ){
  4993   4931     u64 ret = 0;                    /* Return value */
  4994   4932     int iIdx;                       /* For iterating through indexes */
  4995   4933   
  4996         -  ret = fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nTerm);
         4934  +  ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm);
  4997   4935   
  4998   4936     for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){
  4999   4937       int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]);
  5000   4938       if( nByte ){
  5001         -      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nByte);
         4939  +      ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte);
  5002   4940       }
  5003   4941     }
  5004   4942   
  5005   4943     return ret;
  5006   4944   }
  5007   4945   
  5008   4946   /*************************************************************************
................................................................................
  5013   4951   
  5014   4952   /*
  5015   4953   ** Decode a segment-data rowid from the %_data table. This function is
  5016   4954   ** the opposite of macro FTS5_SEGMENT_ROWID().
  5017   4955   */
  5018   4956   static void fts5DecodeRowid(
  5019   4957     i64 iRowid,                     /* Rowid from %_data table */
  5020         -  int *piIdx,                     /* OUT: Index */
  5021   4958     int *piSegid,                   /* OUT: Segment id */
  5022   4959     int *piHeight,                  /* OUT: Height */
  5023   4960     int *piPgno                     /* OUT: Page number */
  5024   4961   ){
  5025   4962     *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1));
  5026   4963     iRowid >>= FTS5_DATA_PAGE_B;
  5027   4964   
  5028   4965     *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1));
  5029   4966     iRowid >>= FTS5_DATA_HEIGHT_B;
  5030   4967   
  5031   4968     *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1));
  5032         -  iRowid >>= FTS5_DATA_ID_B;
  5033         -
  5034         -  *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1));
  5035   4969   }
  5036   4970   
  5037   4971   static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){
  5038         -  int iIdx,iSegid,iHeight,iPgno;  /* Rowid compenents */
  5039         -  fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno);
         4972  +  int iSegid, iHeight, iPgno;     /* Rowid compenents */
         4973  +  fts5DecodeRowid(iKey, &iSegid, &iHeight, &iPgno);
  5040   4974   
  5041   4975     if( iSegid==0 ){
  5042   4976       if( iKey==FTS5_AVERAGES_ROWID ){
  5043   4977         sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) ");
  5044   4978       }else{
  5045   4979         sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 
  5046   4980             "{structure idx=%d}", (int)(iKey-10)
  5047   4981         );
  5048   4982       }
  5049   4983     }
  5050   4984     else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
  5051         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)",
  5052         -        iIdx, iSegid, iPgno
         4985  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx segid=%d pgno=%d)",
         4986  +        iSegid, iPgno
  5053   4987       );
  5054   4988     }else{
  5055         -    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)",
  5056         -        iIdx, iSegid, iHeight, iPgno
         4989  +    sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(segid=%d h=%d pgno=%d)",
         4990  +        iSegid, iHeight, iPgno
  5057   4991       );
  5058   4992     }
  5059   4993   }
  5060   4994   
  5061   4995   static void fts5DebugStructure(
  5062   4996     int *pRc,                       /* IN/OUT: error code */
  5063   4997     Fts5Buffer *pBuf,
................................................................................
  5159   5093   */
  5160   5094   static void fts5DecodeFunction(
  5161   5095     sqlite3_context *pCtx,          /* Function call context */
  5162   5096     int nArg,                       /* Number of args (always 2) */
  5163   5097     sqlite3_value **apVal           /* Function arguments */
  5164   5098   ){
  5165   5099     i64 iRowid;                     /* Rowid for record being decoded */
  5166         -  int iIdx,iSegid,iHeight,iPgno;  /* Rowid components */
         5100  +  int iSegid,iHeight,iPgno;       /* Rowid components */
  5167   5101     const u8 *aBlob; int n;         /* Record to decode */
  5168   5102     u8 *a = 0;
  5169   5103     Fts5Buffer s;                   /* Build up text to return here */
  5170   5104     int rc = SQLITE_OK;             /* Return code */
  5171   5105     int nSpace = 0;
  5172   5106   
  5173   5107     assert( nArg==2 );
................................................................................
  5176   5110     n = sqlite3_value_bytes(apVal[1]);
  5177   5111     aBlob = sqlite3_value_blob(apVal[1]);
  5178   5112   
  5179   5113     nSpace = n + FTS5_DATA_ZERO_PADDING;
  5180   5114     a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace);
  5181   5115     if( a==0 ) goto decode_out;
  5182   5116     memcpy(a, aBlob, n);
  5183         -  fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno);
         5117  +  fts5DecodeRowid(iRowid, &iSegid, &iHeight, &iPgno);
  5184   5118   
  5185   5119     fts5DebugRowid(&rc, &s, iRowid);
  5186   5120     if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){
  5187   5121       Fts5Data dlidx;
  5188   5122       Fts5DlidxIter iter;
  5189   5123   
  5190   5124       dlidx.p = a;
................................................................................
  5297   5231     const char *zArg;
  5298   5232     if( nArg==0 ){
  5299   5233       sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1);
  5300   5234     }else{
  5301   5235       zArg = (const char*)sqlite3_value_text(apVal[0]);
  5302   5236       if( 0==sqlite3_stricmp(zArg, "segment") ){
  5303   5237         i64 iRowid;
  5304         -      int idx, segid, height, pgno;
  5305         -      if( nArg!=5 ){
         5238  +      int segid, height, pgno;
         5239  +      if( nArg!=4 ){
  5306   5240           sqlite3_result_error(pCtx, 
  5307         -            "should be: fts5_rowid('segment', idx, segid, height, pgno))", -1
         5241  +            "should be: fts5_rowid('segment', segid, height, pgno))", -1
  5308   5242           );
  5309   5243         }else{
  5310         -        idx = sqlite3_value_int(apVal[1]);
  5311         -        segid = sqlite3_value_int(apVal[2]);
  5312         -        height = sqlite3_value_int(apVal[3]);
  5313         -        pgno = sqlite3_value_int(apVal[4]);
  5314         -        iRowid = FTS5_SEGMENT_ROWID(idx, segid, height, pgno);
         5244  +        segid = sqlite3_value_int(apVal[1]);
         5245  +        height = sqlite3_value_int(apVal[2]);
         5246  +        pgno = sqlite3_value_int(apVal[3]);
         5247  +        iRowid = FTS5_SEGMENT_ROWID(segid, height, pgno);
  5315   5248           sqlite3_result_int64(pCtx, iRowid);
  5316   5249         }
         5250  +#if 0
  5317   5251       }else if( 0==sqlite3_stricmp(zArg, "start-of-index") ){
  5318   5252         i64 iRowid;
  5319   5253         int idx;
  5320   5254         if( nArg!=2 ){
  5321   5255           sqlite3_result_error(pCtx, 
  5322   5256               "should be: fts5_rowid('start-of-index', idx)", -1
  5323   5257           );
  5324   5258         }else{
  5325   5259           idx = sqlite3_value_int(apVal[1]);
  5326   5260           iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0);
  5327   5261           sqlite3_result_int64(pCtx, iRowid);
  5328   5262         }
         5263  +#endif
  5329   5264       }else {
  5330   5265         sqlite3_result_error(pCtx, 
  5331   5266           "first arg to fts5_rowid() must be 'segment' "
  5332   5267           "or 'start-of-index'"
  5333   5268           , -1
  5334   5269         );
  5335   5270       }

Changes to ext/fts5/fts5_storage.c.

   253    253         );
   254    254       }
   255    255       if( rc==SQLITE_OK ){
   256    256         rc = sqlite3Fts5CreateTable(
   257    257             pConfig, "config", "k PRIMARY KEY, v", 1, pzErr
   258    258         );
   259    259       }
          260  +    if( rc==SQLITE_OK ){
          261  +      rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
          262  +    }
   260    263     }
   261    264   
   262    265     if( rc ){
   263    266       sqlite3Fts5StorageClose(p);
   264    267       *pp = 0;
   265    268     }
   266    269     return rc;
................................................................................
   539    542     );
   540    543   
   541    544     /* Reinitialize the %_data table. This call creates the initial structure
   542    545     ** and averages records.  */
   543    546     if( rc==SQLITE_OK ){
   544    547       rc = sqlite3Fts5IndexReinit(p->pIndex);
   545    548     }
          549  +  if( rc==SQLITE_OK ){
          550  +    rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION);
          551  +  }
   546    552     return rc;
   547    553   }
   548    554   
   549    555   int sqlite3Fts5StorageRebuild(Fts5Storage *p){
   550    556     Fts5Buffer buf = {0,0,0};
   551    557     Fts5Config *pConfig = p->pConfig;
   552    558     sqlite3_stmt *pScan = 0;
................................................................................
   979    985   int sqlite3Fts5StorageRollback(Fts5Storage *p){
   980    986     p->bTotalsValid = 0;
   981    987     return sqlite3Fts5IndexRollback(p->pIndex);
   982    988   }
   983    989   
   984    990   int sqlite3Fts5StorageConfigValue(
   985    991     Fts5Storage *p, 
   986         -  const char *z, 
   987         -  sqlite3_value *pVal
          992  +  const char *z,
          993  +  sqlite3_value *pVal,
          994  +  int iVal
   988    995   ){
   989    996     sqlite3_stmt *pReplace = 0;
   990    997     int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0);
   991    998     if( rc==SQLITE_OK ){
   992    999       sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC);
   993         -    sqlite3_bind_value(pReplace, 2, pVal);
         1000  +    if( pVal ){
         1001  +      sqlite3_bind_value(pReplace, 2, pVal);
         1002  +    }else{
         1003  +      sqlite3_bind_int(pReplace, 2, iVal);
         1004  +    }
   994   1005       sqlite3_step(pReplace);
   995   1006       rc = sqlite3_reset(pReplace);
   996   1007     }
   997         -  if( rc==SQLITE_OK ){
         1008  +  if( rc==SQLITE_OK && pVal ){
   998   1009       int iNew = p->pConfig->iCookie + 1;
   999   1010       rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew);
  1000   1011       if( rc==SQLITE_OK ){
  1001   1012         p->pConfig->iCookie = iNew;
  1002   1013       }
  1003   1014     }
  1004   1015     return rc;
  1005   1016   }
  1006   1017   
  1007   1018   
  1008   1019   #endif /* SQLITE_ENABLE_FTS5 */

Changes to ext/fts5/test/fts5al.test.

    22     22     finish_test
    23     23     return
    24     24   }
    25     25   
    26     26   do_execsql_test 1.1 {
    27     27     CREATE VIRTUAL TABLE ft1 USING fts5(x);
    28     28     SELECT * FROM ft1_config;
    29         -} {}
           29  +} {version 1}
    30     30   
    31     31   do_execsql_test 1.2 {
    32     32     INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32);
    33     33     SELECT * FROM ft1_config;
    34         -} {pgsz 32}
           34  +} {pgsz 32 version 1}
    35     35   
    36     36   do_execsql_test 1.3 {
    37     37     INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64);
    38     38     SELECT * FROM ft1_config;
    39         -} {pgsz 64}
           39  +} {pgsz 64 version 1}
    40     40   
    41     41   #--------------------------------------------------------------------------
    42     42   # Test the logic for parsing the rank() function definition.
    43     43   #
    44     44   foreach {tn defn} {
    45     45     1 "fname()"
    46     46     2 "fname(1)"

Changes to ext/fts5/test/fts5corrupt.test.

    33     33   db_save
    34     34   
    35     35   do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
    36     36   set segid [lindex [fts5_level_segids t1] 0]
    37     37   
    38     38   do_test 1.3 {
    39     39     execsql {
    40         -    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4);
           40  +    DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 0, 4);
    41     41     }
    42     42     catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    43     43   } {1 {database disk image is malformed}}
    44     44   
    45     45   do_test 1.4 {
    46     46     db_restore_and_reopen
    47     47     execsql {
    48     48       UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE
    49         -    rowid = fts5_rowid('segment', 0, $segid, 0, 4);
           49  +    rowid = fts5_rowid('segment', $segid, 0, 4);
    50     50     }
    51     51     catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
    52     52   } {1 {database disk image is malformed}}
    53     53   
    54     54   db_restore_and_reopen
    55     55   #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
    56     56   

Changes to ext/fts5/test/fts5corrupt2.test.

   122    122     INSERT INTO x3(x3, rank) VALUES('pgsz', 32);
   123    123     WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000) 
   124    124     INSERT INTO x3 
   125    125     SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
   126    126   }
   127    127   
   128    128   foreach {tn hdr} {
   129         -  1 "\00\00\00\00"
   130         -  2 "\FF\FF\FF\FF"
          129  +  1 "\x00\x00\x00\x00"
          130  +  2 "\xFF\xFF\xFF\xFF"
   131    131   } {
   132    132     set tn2 0
   133    133     set nCorrupt 0
   134    134     foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
   135    135       if {$rowid & $mask} continue
   136    136       incr tn2
   137    137       do_test 3.$tn.$tn2 {

Changes to ext/fts5/test/fts5ea.test.

    74     74     11 {a AND "abc}                  {unterminated string}
    75     75   
    76     76     12 {NEAR(a b, xyz)}              {expected integer, got "xyz"}
    77     77     13 {NEAR(a b, // )}              {expected integer, got "//"}
    78     78   } {
    79     79     do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err]
    80     80   }
           81  +
           82  +#-------------------------------------------------------------------------
           83  +# Experiment with a tokenizer that considers " to be a token character.
           84  +#
           85  +do_execsql_test 4.0 {
           86  +  SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"');
           87  +} {{"a" AND """"}}
    81     88   
    82     89   
    83     90   
    84     91   
    85     92   finish_test

Changes to ext/fts5/test/fts5fault1.test.

    28     28   #   2: INSERT statement
    29     29   #   3: DELETE statement
    30     30   #   4: MATCH expressions
    31     31   #
    32     32   #
    33     33   
    34     34   faultsim_save_and_close
    35         -do_faultsim_test 1 -prep {
           35  +do_faultsim_test 1 -faults ioerr-t* -prep {
    36     36     faultsim_restore_and_reopen
    37     37   } -body {
    38     38     execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') }
    39     39   } -test {
    40         -  faultsim_test_result {0 {}} 
           40  +  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}}
    41     41   }
    42     42   
    43     43   reset_db
    44     44   do_execsql_test 2.0 {
    45     45     CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3');
    46     46   }
    47     47   faultsim_save_and_close

Changes to ext/fts5/test/fts5fault4.test.

   197    197   
   198    198   do_faultsim_test 6.2 -faults oom-t* -body {
   199    199     db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' }
   200    200   } -test {
   201    201     faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM}
   202    202   }
   203    203   
   204         -}
   205         -
   206    204   #-------------------------------------------------------------------------
   207    205   # OOM error when querying for a phrase with many tokens.
   208    206   #
   209    207   reset_db
   210    208   do_execsql_test 7.0 {
   211    209     CREATE VIRTUAL TABLE tt USING fts5(x, y);
   212    210     INSERT INTO tt VALUES('f b g b c b', 'f a d c c b');  -- 1
................................................................................
   299    297   }
   300    298   
   301    299   do_faultsim_test 9.1 -faults oom-* -body {
   302    300     db eval { SELECT rowid FROM tt WHERE tt MATCH 'a NOT x' }
   303    301   } -test {
   304    302     faultsim_test_result {0 {50 100 150 200}} {1 SQLITE_NOMEM}
   305    303   }
          304  +
          305  +#-------------------------------------------------------------------------
          306  +# OOM in fts5_expr() SQL function.
          307  +#
          308  +reset_db
          309  +do_execsql_test 10.0 {
          310  +  CREATE VIRTUAL TABLE tt USING fts5(x);
          311  +  INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
          312  +  BEGIN;
          313  +    WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<200)
          314  +      INSERT INTO tt(rowid, x) 
          315  +      SELECT i, CASE WHEN (i%50)==0 THEN 'a a a a a a' ELSE 'a x a x a x' END 
          316  +      FROM ii;
          317  +  COMMIT;
          318  +}
          319  +
          320  +}
          321  +
          322  +do_faultsim_test 10.1 -faults oom-t* -body {
          323  +  db one { SELECT fts5_expr('a AND b NEAR(a b)') }
          324  +} -test {
          325  +  faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}} 
          326  +}
          327  +
          328  +#do_faultsim_test 10.2 -faults oom-t* -body {
          329  +#  db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') }
          330  +#} -test {
          331  +#  set res {[ns -col 0 -- {a b c}] && ([ns -- {b}] && [ns -near 10 -- {a} {b}]}
          332  +#  faultsim_test_result [list 0 $res]
          333  +#}
   306    334   
   307    335   
   308    336   
   309    337   finish_test
   310    338   

Added ext/fts5/test/fts5integrity.test.

            1  +# 2015 Jan 13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#***********************************************************************
           11  +#
           12  +# This file containst tests focused on the integrity-check procedure.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5integrity
           17  +
           18  +do_execsql_test 1.0 {
           19  +  CREATE VIRTUAL TABLE xx USING fts5(x);
           20  +  INSERT INTO xx VALUES('term');
           21  +}
           22  +do_execsql_test 1.1 {
           23  +  INSERT INTO xx(xx) VALUES('integrity-check');
           24  +}
           25  +
           26  +do_execsql_test 2.0 {
           27  +  CREATE VIRTUAL TABLE yy USING fts5(x, prefix=1);
           28  +  INSERT INTO yy VALUES('term');
           29  +}
           30  +do_execsql_test 2.1 {
           31  +  INSERT INTO yy(yy) VALUES('integrity-check');
           32  +}
           33  +
           34  +finish_test
           35  +

Changes to ext/fts5/test/fts5prefix.test.

    11     11   #
    12     12   # This file containst tests focused on prefix indexes.
    13     13   #
    14     14   
    15     15   source [file join [file dirname [info script]] fts5_common.tcl]
    16     16   set testprefix fts5prefix
    17     17   
           18  +do_execsql_test 1.0 {
           19  +  CREATE VIRTUAL TABLE xx USING fts5(x, prefix=1);
           20  +  INSERT INTO xx VALUES('one two three');
           21  +  INSERT INTO xx VALUES('four five six');
           22  +  INSERT INTO xx VALUES('seven eight nine ten');
           23  +}
           24  +
           25  +do_execsql_test 1.1 {
           26  +  SELECT rowid FROM xx WHERE xx MATCH 't*'
           27  +} {1 3}
           28  +
    18     29   
    19     30   #-------------------------------------------------------------------------
    20     31   # Check that prefix indexes really do index n-character prefixes, not 
    21     32   # n-byte prefixes. Use the ascii tokenizer so as not to be confused by
    22     33   # diacritic removal.
    23     34   #
    24         -do_execsql_test 1.0 { 
           35  +do_execsql_test 2.0 { 
    25     36     CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = ascii, prefix = 2) 
    26     37   }
    27     38   
    28         -do_test 1.2 {
           39  +do_test 2.1 {
    29     40     foreach {rowid string} {
    30     41       1 "\xCA\xCB\xCC\xCD"
    31     42       2 "\u1234\u5678\u4321\u8765"
    32     43     } {
    33     44       execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $string) }
    34     45     }
    35     46   } {}
    36     47   
    37         -do_execsql_test 1.1.2 {
           48  +do_execsql_test 2.2 {
    38     49     INSERT INTO t1(t1) VALUES('integrity-check');
    39     50   }
    40     51   
    41         -#db eval { select fts5_decode(id, block) AS d FROM t1_data; } { puts $d }
    42         -
    43         -foreach o {1 2} {
    44         -  if {$o==2} breakpoint
    45         -  foreach {tn q res} {
    46         -    1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1
    47         -    2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2
    48         -  } {
    49         -    do_execsql_test 1.$o.$tn $q $res
    50         -  }
    51         -
    52         -  execsql {
    53         -    DELETE FROM t1_data WHERE 
    54         -    rowid>=fts5_rowid('start-of-index', 0) AND 
    55         -    rowid<fts5_rowid('start-of-index', 1);
    56         -  }
           52  +foreach {tn q res} {
           53  +  1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1
           54  +  2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2
           55  +} {
           56  +  do_execsql_test 2.3.$tn $q $res
    57     57   }
    58     58   
    59     59   
    60     60   finish_test
    61     61   

Changes to ext/fts5/test/fts5rowid.test.

    17     17   
    18     18   do_catchsql_test 1.1 {
    19     19     SELECT fts5_rowid()
    20     20   } {1 {should be: fts5_rowid(subject, ....)}}
    21     21   
    22     22   do_catchsql_test 1.2 {
    23     23     SELECT fts5_rowid('segment')
    24         -} {1 {should be: fts5_rowid('segment', idx, segid, height, pgno))}}
           24  +} {1 {should be: fts5_rowid('segment', segid, height, pgno))}}
    25     25   
    26     26   do_execsql_test 1.3 {
    27         -  SELECT fts5_rowid('segment', 1, 1, 1, 1)
    28         -} {4503670494330881}
    29         -
    30         -do_catchsql_test 1.4 {
    31         -  SELECT fts5_rowid('start-of-index');
    32         -} {1 {should be: fts5_rowid('start-of-index', idx)}}
    33         -
    34         -do_execsql_test 1.5 {
    35         -  SELECT fts5_rowid('start-of-index', 1);
    36         -} {4503668346847232}
           27  +  SELECT fts5_rowid('segment', 1, 1, 1)
           28  +} {70866960385}
    37     29   
    38     30   do_catchsql_test 1.4 {
    39     31     SELECT fts5_rowid('nosucharg');
    40     32   } {1 {first arg to fts5_rowid() must be 'segment' or 'start-of-index'}} 
    41     33   
    42     34   
    43     35   #-------------------------------------------------------------------------

Changes to ext/fts5/tool/loadfts5.tcl.

    36     36     puts stderr "  -fts4        (use fts4 instead of fts5)"
    37     37     puts stderr "  -fts5        (use fts5)"
    38     38     puts stderr "  -porter      (use porter tokenizer)"
    39     39     puts stderr "  -delete      (delete the database file before starting)"
    40     40     puts stderr "  -limit N     (load no more than N documents)"
    41     41     puts stderr "  -automerge N (set the automerge parameter to N)"
    42     42     puts stderr "  -crisismerge N (set the crisismerge parameter to N)"
           43  +  puts stderr "  -prefix PREFIX (comma separated prefix= argument)"
    43     44     exit 1
    44     45   }
    45     46   
    46     47   set O(vtab)       fts5
    47     48   set O(tok)        ""
    48     49   set O(limit)      0
    49     50   set O(delete)     0
    50     51   set O(automerge)  -1
    51     52   set O(crisismerge)  -1
           53  +set O(prefix)     ""
    52     54   
    53     55   if {[llength $argv]<2} usage
    54     56   set nOpt [expr {[llength $argv]-2}]
    55     57   for {set i 0} {$i < $nOpt} {incr i} {
    56     58     set arg [lindex $argv $i]
    57     59     switch -- [lindex $argv $i] {
    58     60       -fts4 {
................................................................................
    81     83         set O(automerge) [lindex $argv $i]
    82     84       }
    83     85   
    84     86       -crisismerge {
    85     87         if { [incr i]>=$nOpt } usage
    86     88         set O(crisismerge) [lindex $argv $i]
    87     89       }
           90  +
           91  +    -prefix {
           92  +      if { [incr i]>=$nOpt } usage
           93  +      set O(prefix) [lindex $argv $i]
           94  +    }
    88     95   
    89     96       default {
    90     97         usage
    91     98       }
    92     99     }
    93    100   }
    94    101   
    95    102   set dbfile [lindex $argv end-1]
    96    103   if {$O(delete)} { file delete -force $dbfile }
    97    104   sqlite3 db $dbfile
    98    105   db func loadfile loadfile
    99    106   
   100    107   db transaction {
          108  +  set pref ""
          109  +  if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" }
   101    110     catch {
   102         -    db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))"
          111  +    db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)"
   103    112     }
   104    113     if {$O(automerge)>=0} {
   105    114       if {$O(vtab) == "fts5"} {
   106    115         db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) }
   107    116       } else {
   108    117         db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) }
   109    118       }