Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Change to storing all keys in a single merge-tree structure instead of one main structure and a separate one for each prefix index. This is a file-format change. Also introduce a mechanism for managing file-format changes. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts5 |
| Files: | files | file ages | folders |
| SHA1: |
a684b5e2d9d52cf4700e7e5f9dd547a2 |
| User & Date: | dan 2015-05-07 19:29:46 |
Context
|
2015-05-08
| ||
| 09:21 | Improve the error message returned by FTS5 if it encounters an unknown file format. check-in: f369caec14 user: dan tags: fts5 | |
|
2015-05-07
| ||
| 19:29 | Change to storing all keys in a single merge-tree structure instead of one main structure and a separate one for each prefix index. This is a file-format change. Also introduce a mechanism for managing file-format changes. check-in: a684b5e2d9 user: dan tags: fts5 | |
|
2015-05-02
| ||
| 20:35 | Reorganize some of the fts5 expression parsing code. Improve test coverage of the same. check-in: c4456dc5f5 user: dan tags: fts5 | |
Changes
Changes to ext/fts5/fts5.c.
1160 1160 if( rc==SQLITE_OK ){ 1161 1161 rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError); 1162 1162 } 1163 1163 if( rc==SQLITE_OK ){ 1164 1164 if( bError ){ 1165 1165 rc = SQLITE_ERROR; 1166 1166 }else{ 1167 - rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal); 1167 + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal, 0); 1168 1168 } 1169 1169 } 1170 1170 } 1171 1171 return rc; 1172 1172 } 1173 1173 1174 1174 static int fts5SpecialDelete(
Changes to ext/fts5/fts5Int.h.
117 117 int iCookie; /* Incremented when %_config is modified */ 118 118 int pgsz; /* Approximate page size used in %_data */ 119 119 int nAutomerge; /* 'automerge' setting */ 120 120 int nCrisisMerge; /* Maximum allowed segments per level */ 121 121 char *zRank; /* Name of rank function */ 122 122 char *zRankArgs; /* Arguments to rank function */ 123 123 }; 124 + 125 +/* Current expected value of %_config table 'version' field */ 126 +#define FTS5_CURRENT_VERSION 1 124 127 125 128 #define FTS5_CONTENT_NORMAL 0 126 129 #define FTS5_CONTENT_NONE 1 127 130 #define FTS5_CONTENT_EXTERNAL 2 131 + 128 132 129 133 130 134 131 135 int sqlite3Fts5ConfigParse( 132 136 Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** 133 137 ); 134 138 void sqlite3Fts5ConfigFree(Fts5Config*); ................................................................................ 390 394 void sqlite3Fts5HashFree(Fts5Hash*); 391 395 392 396 int sqlite3Fts5HashWrite( 393 397 Fts5Hash*, 394 398 i64 iRowid, /* Rowid for this entry */ 395 399 int iCol, /* Column token appears in (-ve -> delete) */ 396 400 int iPos, /* Position of token within column */ 401 + char bByte, 397 402 const char *pToken, int nToken /* Token to add or remove to or from index */ 398 403 ); 399 404 400 405 /* 401 406 ** Empty (but do not delete) a hash table. 402 407 */ 403 408 void sqlite3Fts5HashClear(Fts5Hash*); ................................................................................ 454 459 int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); 455 460 int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); 456 461 int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); 457 462 458 463 int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); 459 464 int sqlite3Fts5StorageRollback(Fts5Storage *p); 460 465 461 -int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*); 466 +int sqlite3Fts5StorageConfigValue( 467 + Fts5Storage *p, const char*, sqlite3_value*, int 468 +); 462 469 463 470 int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); 464 471 465 472 int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); 466 473 int sqlite3Fts5StorageRebuild(Fts5Storage *p); 467 474 int sqlite3Fts5StorageOptimize(Fts5Storage *p); 468 475 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
Changes to ext/fts5/fts5_config.c.
199 199 assert( 0==fts5_iswhitespace(z[0]) ); 200 200 quote = z[0]; 201 201 if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ 202 202 fts5Dequote(z); 203 203 } 204 204 } 205 205 206 -/* 207 -** Argument z points to a nul-terminated string containing an SQL identifier. 208 -** This function returns a copy of the identifier enclosed in backtick 209 -** quotes. 210 -*/ 211 -static char *fts5EscapeName(int *pRc, const char *z){ 212 - char *pRet = 0; 213 - if( *pRc==SQLITE_OK ){ 214 - int n = strlen(z); 215 - pRet = (char*)sqlite3_malloc(2 + 2*n + 1); 216 - if( pRet==0 ){ 217 - *pRc = SQLITE_NOMEM; 218 - }else{ 219 - int i; 220 - char *p = pRet; 221 - *p++ = '`'; 222 - for(i=0; i<n; i++){ 223 - if( z[i]=='`' ) *p++ = '`'; 224 - *p++ = z[i]; 225 - } 226 - *p++ = '`'; 227 - *p++ = '\0'; 228 - } 229 - } 230 - return pRet; 231 -} 232 - 233 206 /* 234 207 ** Parse the "special" CREATE VIRTUAL TABLE directive and update 235 208 ** configuration object pConfig as appropriate. 236 209 ** 237 210 ** If successful, object pConfig is updated and SQLITE_OK returned. If 238 211 ** an error occurs, an SQLite error code is returned and an error message 239 212 ** may be left in *pzErr. It is the responsibility of the caller to ................................................................................ 455 428 /* 456 429 ** Populate the Fts5Config.zContentExprlist string. 457 430 */ 458 431 static int fts5ConfigMakeExprlist(Fts5Config *p){ 459 432 int i; 460 433 int rc = SQLITE_OK; 461 434 Fts5Buffer buf = {0, 0, 0}; 462 - const char *zSep = ""; 463 435 464 436 sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); 465 437 if( p->eContent!=FTS5_CONTENT_NONE ){ 466 438 for(i=0; i<p->nCol; i++){ 467 439 if( p->eContent==FTS5_CONTENT_EXTERNAL ){ 468 440 sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); 469 441 }else{ ................................................................................ 845 817 ** Load the contents of the %_config table into memory. 846 818 */ 847 819 int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ 848 820 const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; 849 821 char *zSql; 850 822 sqlite3_stmt *p = 0; 851 823 int rc; 824 + int iVersion = 0; 852 825 853 826 /* Set default values */ 854 827 pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; 855 828 pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; 856 829 pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; 857 830 858 831 zSql = sqlite3_mprintf(zSelect, pConfig->zDb, pConfig->zName); ................................................................................ 864 837 } 865 838 866 839 assert( rc==SQLITE_OK || p==0 ); 867 840 if( rc==SQLITE_OK ){ 868 841 while( SQLITE_ROW==sqlite3_step(p) ){ 869 842 const char *zK = (const char*)sqlite3_column_text(p, 0); 870 843 sqlite3_value *pVal = sqlite3_column_value(p, 1); 871 - sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0); 844 + if( 0==sqlite3_stricmp(zK, "version") ){ 845 + iVersion = sqlite3_value_int(pVal); 846 + }else{ 847 + sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0); 848 + } 872 849 } 873 - rc = sqlite3_finalize(p); 850 + if( rc==SQLITE_OK ) rc = sqlite3_finalize(p); 851 + } 852 + 853 + if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ 854 + rc = sqlite3Fts5Corrupt(); 874 855 } 875 856 876 857 if( rc==SQLITE_OK ){ 877 858 pConfig->iCookie = iCookie; 878 859 } 879 860 return rc; 880 861 } 881 862 882 863 #endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/fts5_expr.c.
1367 1367 1368 1368 static char *fts5PrintfAppend(char *zApp, const char *zFmt, ...){ 1369 1369 char *zNew; 1370 1370 va_list ap; 1371 1371 va_start(ap, zFmt); 1372 1372 zNew = sqlite3_vmprintf(zFmt, ap); 1373 1373 va_end(ap); 1374 - if( zApp ){ 1374 + if( zApp && zNew ){ 1375 1375 char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew); 1376 1376 sqlite3_free(zNew); 1377 1377 zNew = zNew2; 1378 1378 } 1379 1379 sqlite3_free(zApp); 1380 1380 return zNew; 1381 1381 } ................................................................................ 1544 1544 int rc; 1545 1545 int i; 1546 1546 1547 1547 const char **azConfig; /* Array of arguments for Fts5Config */ 1548 1548 const char *zNearsetCmd = "nearset"; 1549 1549 int nConfig; /* Size of azConfig[] */ 1550 1550 Fts5Config *pConfig = 0; 1551 + int iArg = 1; 1551 1552 1552 1553 if( bTcl && nArg>1 ){ 1553 1554 zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); 1555 + iArg = 2; 1554 1556 } 1555 1557 1556 - nConfig = nArg + 2 - bTcl; 1558 + nConfig = 3 + (nArg-iArg); 1557 1559 azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig); 1558 1560 if( azConfig==0 ){ 1559 1561 sqlite3_result_error_nomem(pCtx); 1560 1562 return; 1561 1563 } 1562 1564 azConfig[0] = 0; 1563 1565 azConfig[1] = "main"; 1564 1566 azConfig[2] = "tbl"; 1565 - for(i=1+bTcl; i<nArg; i++){ 1566 - azConfig[i+2-bTcl] = (const char*)sqlite3_value_text(apVal[i]); 1567 + for(i=3; iArg<nArg; iArg++){ 1568 + azConfig[i++] = (const char*)sqlite3_value_text(apVal[iArg]); 1567 1569 } 1570 + 1568 1571 zExpr = (const char*)sqlite3_value_text(apVal[0]); 1569 1572 1570 1573 rc = sqlite3Fts5ConfigParse(pGlobal, db, nConfig, azConfig, &pConfig, &zErr); 1571 1574 if( rc==SQLITE_OK ){ 1572 1575 rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pExpr, &zErr); 1573 1576 } 1574 1577 if( rc==SQLITE_OK ){ ................................................................................ 1576 1579 if( pExpr->pRoot==0 ){ 1577 1580 zText = sqlite3_mprintf(""); 1578 1581 }else if( bTcl ){ 1579 1582 zText = fts5ExprPrintTcl(pConfig, zNearsetCmd, pExpr->pRoot); 1580 1583 }else{ 1581 1584 zText = fts5ExprPrint(pConfig, pExpr->pRoot); 1582 1585 } 1583 - if( rc==SQLITE_OK ){ 1586 + if( zText==0 ){ 1587 + rc = SQLITE_NOMEM; 1588 + }else{ 1584 1589 sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT); 1585 1590 sqlite3_free(zText); 1586 1591 } 1587 1592 } 1588 1593 1589 1594 if( rc!=SQLITE_OK ){ 1590 1595 if( zErr ){
Changes to ext/fts5/fts5_hash.c.
131 131 int i; 132 132 unsigned int h = 13; 133 133 for(i=n-1; i>=0; i--){ 134 134 h = (h << 3) ^ h ^ p[i]; 135 135 } 136 136 return (h % nSlot); 137 137 } 138 + 139 +static unsigned int fts5HashKey2(int nSlot, char b, const char *p, int n){ 140 + int i; 141 + unsigned int h = 13; 142 + for(i=n-1; i>=0; i--){ 143 + h = (h << 3) ^ h ^ p[i]; 144 + } 145 + h = (h << 3) ^ h ^ b; 146 + return (h % nSlot); 147 +} 138 148 139 149 /* 140 150 ** Resize the hash table by doubling the number of slots. 141 151 */ 142 152 static int fts5HashResize(Fts5Hash *pHash){ 143 153 int nNew = pHash->nSlot*2; 144 154 int i; ................................................................................ 187 197 } 188 198 189 199 int sqlite3Fts5HashWrite( 190 200 Fts5Hash *pHash, 191 201 i64 iRowid, /* Rowid for this entry */ 192 202 int iCol, /* Column token appears in (-ve -> delete) */ 193 203 int iPos, /* Position of token within column */ 204 + char bByte, /* First byte of token */ 194 205 const char *pToken, int nToken /* Token to add or remove to or from index */ 195 206 ){ 196 - unsigned int iHash = fts5HashKey(pHash->nSlot, pToken, nToken); 207 + unsigned int iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); 197 208 Fts5HashEntry *p; 198 209 u8 *pPtr; 199 210 int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ 200 211 201 212 /* Attempt to locate an existing hash entry */ 202 213 for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ 203 - if( memcmp(p->zKey, pToken, nToken)==0 && p->zKey[nToken]==0 ) break; 214 + if( p->zKey[0]==bByte 215 + && memcmp(&p->zKey[1], pToken, nToken)==0 216 + && p->zKey[nToken+1]==0 217 + ){ 218 + break; 219 + } 204 220 } 205 221 206 222 /* If an existing hash entry cannot be found, create a new one. */ 207 223 if( p==0 ){ 208 - int nByte = sizeof(Fts5HashEntry) + nToken + 1 + 64; 224 + int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; 209 225 if( nByte<128 ) nByte = 128; 210 226 211 227 if( (pHash->nEntry*2)>=pHash->nSlot ){ 212 228 int rc = fts5HashResize(pHash); 213 229 if( rc!=SQLITE_OK ) return rc; 214 - iHash = fts5HashKey(pHash->nSlot, pToken, nToken); 230 + iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); 215 231 } 216 232 217 233 p = (Fts5HashEntry*)sqlite3_malloc(nByte); 218 234 if( !p ) return SQLITE_NOMEM; 219 235 memset(p, 0, sizeof(Fts5HashEntry)); 220 236 p->nAlloc = nByte; 221 - memcpy(p->zKey, pToken, nToken); 222 - p->zKey[nToken] = '\0'; 223 - p->nData = nToken + 1 + sizeof(Fts5HashEntry); 237 + p->zKey[0] = bByte; 238 + memcpy(&p->zKey[1], pToken, nToken); 239 + assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) ); 240 + p->zKey[nToken+1] = '\0'; 241 + p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); 224 242 p->nData += sqlite3PutVarint(&((u8*)p)[p->nData], iRowid); 225 243 p->iSzPoslist = p->nData; 226 244 p->nData += 1; 227 245 p->iRowid = iRowid; 228 246 p->pHashNext = pHash->aSlot[iHash]; 229 247 pHash->aSlot[iHash] = p; 230 248 pHash->nEntry++;
Changes to ext/fts5/fts5_index.c.
45 45 46 46 47 47 #define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ 48 48 #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ 49 49 50 50 #define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ 51 51 52 +#define FTS5_MAIN_PREFIX '0' 53 + 54 +#if FTS5_MAX_PREFIX_INDEXES > 31 55 +# error "FTS5_MAX_PREFIX_INDEXES is too large" 56 +#endif 57 + 52 58 /* 53 59 ** Details: 54 60 ** 55 61 ** The %_data table managed by this module, 56 62 ** 57 63 ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); 58 64 ** ................................................................................ 207 213 ** representing the first docid on the page otherwise. 208 214 */ 209 215 210 216 /* 211 217 ** Rowids for the averages and structure records in the %_data table. 212 218 */ 213 219 #define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ 214 -#define FTS5_STRUCTURE_ROWID(iIdx) (10 + (iIdx)) /* For structure records */ 220 +#define FTS5_STRUCTURE_ROWID 10 /* The structure record */ 215 221 216 222 /* 217 223 ** Macros determining the rowids used by segment nodes. All nodes in all 218 224 ** segments for all indexes (the regular FTS index and any prefix indexes) 219 225 ** are stored in the %_data table with large positive rowids. 220 226 ** 221 227 ** The %_data table may contain up to (1<<FTS5_SEGMENT_INDEX_BITS) ................................................................................ 229 235 ** a nodes page number is always one more than its left sibling. 230 236 ** 231 237 ** The rowid for a node is then found using the FTS5_SEGMENT_ROWID() macro 232 238 ** below. The FTS5_SEGMENT_*_BITS macros define the number of bits used 233 239 ** to encode the three FTS5_SEGMENT_ROWID() arguments. This module returns 234 240 ** SQLITE_FULL and fails the current operation if they ever prove too small. 235 241 */ 236 -#define FTS5_DATA_IDX_B 5 /* Max of 31 prefix indexes */ 237 242 #define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ 238 243 #define FTS5_DATA_HEIGHT_B 5 /* Max b-tree height of 32 */ 239 244 #define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ 240 245 241 -#define FTS5_SEGMENT_ROWID(idx, segid, height, pgno) ( \ 242 - ((i64)(idx) << (FTS5_DATA_ID_B + FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ 246 +#define FTS5_SEGMENT_ROWID(segid, height, pgno) ( \ 243 247 ((i64)(segid) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ 244 248 ((i64)(height) << (FTS5_DATA_PAGE_B)) + \ 245 249 ((i64)(pgno)) \ 246 250 ) 247 251 248 -#if FTS5_MAX_PREFIX_INDEXES > ((1<<FTS5_DATA_IDX_B)-1) 249 -# error "FTS5_MAX_PREFIX_INDEXES is too large" 250 -#endif 251 - 252 252 /* 253 253 ** The height of segment b-trees is actually limited to one less than 254 254 ** (1<<HEIGHT_BITS). This is because the rowid address space for nodes 255 255 ** with such a height is used by doclist indexes. 256 256 */ 257 257 #define FTS5_SEGMENT_MAX_HEIGHT ((1 << FTS5_DATA_HEIGHT_B)-1) 258 258 ................................................................................ 261 261 */ 262 262 #define FTS5_MAX_SEGMENT 2000 263 263 264 264 /* 265 265 ** The rowid for the doclist index associated with leaf page pgno of segment 266 266 ** segid in index idx. 267 267 */ 268 -#define FTS5_DOCLIST_IDX_ROWID(idx, segid, pgno) \ 269 - FTS5_SEGMENT_ROWID(idx, segid, FTS5_SEGMENT_MAX_HEIGHT, pgno) 268 +#define FTS5_DOCLIST_IDX_ROWID(segid, pgno) \ 269 + FTS5_SEGMENT_ROWID(segid, FTS5_SEGMENT_MAX_HEIGHT, pgno) 270 270 271 271 #ifdef SQLITE_DEBUG 272 272 int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } 273 273 #endif 274 274 275 275 276 276 /* ................................................................................ 310 310 char *zDataTbl; /* Name of %_data table */ 311 311 int nWorkUnit; /* Leaf pages in a "unit" of work */ 312 312 313 313 /* 314 314 ** Variables related to the accumulation of tokens and doclists within the 315 315 ** in-memory hash tables before they are flushed to disk. 316 316 */ 317 - Fts5Hash **apHash; /* Array of hash tables */ 317 + Fts5Hash *pHash; /* Hash table for in-memory data */ 318 318 int nMaxPendingData; /* Max pending data before flush to disk */ 319 319 int nPendingData; /* Current bytes of pending data */ 320 320 i64 iWriteRowid; /* Rowid for current doc being written */ 321 + Fts5Buffer scratch; 321 322 322 323 /* Error state. */ 323 324 int rc; /* Current error code */ 324 325 325 326 /* State used by the fts5DataXXX() functions. */ 326 327 sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ 327 328 sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ ................................................................................ 380 381 */ 381 382 struct Fts5PageWriter { 382 383 int pgno; /* Page number for this page */ 383 384 Fts5Buffer buf; /* Buffer containing page data */ 384 385 Fts5Buffer term; /* Buffer containing previous term on page */ 385 386 }; 386 387 struct Fts5SegWriter { 387 - int iIdx; /* Index to write to */ 388 388 int iSegid; /* Segid to write to */ 389 389 int nWriter; /* Number of entries in aWriter */ 390 390 Fts5PageWriter *aWriter; /* Array of PageWriter objects */ 391 391 i64 iPrevRowid; /* Previous docid written to current leaf */ 392 392 u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ 393 393 u8 bFirstRowidInPage; /* True if next rowid is first in page */ 394 394 u8 bFirstTermInPage; /* True if next term will be first in leaf */ ................................................................................ 474 474 ** 475 475 ** For each rowid on the page corresponding to the current term, the 476 476 ** corresponding aRowidOffset[] entry is set to the byte offset of the 477 477 ** start of the "position-list-size" field within the page. 478 478 */ 479 479 struct Fts5SegIter { 480 480 Fts5StructureSegment *pSeg; /* Segment to iterate through */ 481 - int iIdx; /* Byte offset within current leaf */ 482 481 int flags; /* Mask of configuration flags */ 483 482 int iLeafPgno; /* Current leaf page number */ 484 483 Fts5Data *pLeaf; /* Current leaf data */ 485 484 int iLeafOffset; /* Byte offset within current leaf */ 486 485 487 486 /* The page and offset from which the current term was read. The offset 488 487 ** is the offset of the first rowid in the current doclist. */ ................................................................................ 595 594 struct Fts5BtreeIterLevel { 596 595 Fts5NodeIter s; /* Iterator for the current node */ 597 596 Fts5Data *pData; /* Data for the current node */ 598 597 }; 599 598 struct Fts5BtreeIter { 600 599 Fts5Index *p; /* FTS5 backend object */ 601 600 Fts5StructureSegment *pSeg; /* Iterate through this segment's b-tree */ 602 - int iIdx; /* Index pSeg belongs to */ 603 601 int nLvl; /* Size of aLvl[] array */ 604 602 Fts5BtreeIterLevel *aLvl; /* Level for each tier of b-tree */ 605 603 606 604 /* Output variables */ 607 605 Fts5Buffer term; /* Current term */ 608 606 int iLeaf; /* Leaf containing terms >= current term */ 609 607 int nEmpty; /* Number of "empty" leaves following iLeaf */ ................................................................................ 987 985 sqlite3_blob_close(p->pReader); 988 986 p->pReader = 0; 989 987 } 990 988 } 991 989 #endif 992 990 993 991 /* 994 -** Remove all records associated with segment iSegid in index iIdx. 992 +** Remove all records associated with segment iSegid. 995 993 */ 996 -static void fts5DataRemoveSegment(Fts5Index *p, int iIdx, int iSegid){ 997 - i64 iFirst = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, 0); 998 - i64 iLast = FTS5_SEGMENT_ROWID(iIdx, iSegid+1, 0, 0)-1; 994 +static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ 995 + i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0); 996 + i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1; 999 997 fts5DataDelete(p, iFirst, iLast); 1000 998 } 1001 999 1002 1000 /* 1003 1001 ** Release a reference to an Fts5Structure object returned by an earlier 1004 1002 ** call to fts5StructureRead() or fts5StructureDecode(). 1005 1003 */ ................................................................................ 1142 1140 }else{ 1143 1141 *pRc = SQLITE_NOMEM; 1144 1142 } 1145 1143 } 1146 1144 } 1147 1145 1148 1146 /* 1149 -** Read, deserialize and return the structure record for index iIdx. 1147 +** Read, deserialize and return the structure record. 1150 1148 ** 1151 1149 ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array 1152 1150 ** are over-allocated as described for function fts5StructureDecode() 1153 1151 ** above. 1154 1152 ** 1155 1153 ** If an error occurs, NULL is returned and an error code left in the 1156 1154 ** Fts5Index handle. If an error has already occurred when this function 1157 1155 ** is called, it is a no-op. 1158 1156 */ 1159 -static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){ 1157 +static Fts5Structure *fts5StructureRead(Fts5Index *p){ 1160 1158 Fts5Config *pConfig = p->pConfig; 1161 1159 Fts5Structure *pRet = 0; /* Object to return */ 1162 1160 Fts5Data *pData; /* %_data entry containing structure record */ 1163 1161 int iCookie; /* Configuration cookie */ 1164 1162 1165 - assert( iIdx<=pConfig->nPrefix ); 1166 - pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx)); 1163 + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); 1167 1164 if( !pData ) return 0; 1168 1165 p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); 1169 1166 1170 1167 if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ 1171 1168 p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); 1172 1169 } 1173 1170 ................................................................................ 1194 1191 } 1195 1192 1196 1193 return nSegment; 1197 1194 } 1198 1195 #endif 1199 1196 1200 1197 /* 1201 -** Serialize and store the "structure" record for index iIdx. 1198 +** Serialize and store the "structure" record. 1202 1199 ** 1203 1200 ** If an error occurs, leave an error code in the Fts5Index object. If an 1204 1201 ** error has already occurred, this function is a no-op. 1205 1202 */ 1206 -static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){ 1203 +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ 1207 1204 if( p->rc==SQLITE_OK ){ 1208 1205 Fts5Buffer buf; /* Buffer to serialize record into */ 1209 1206 int iLvl; /* Used to iterate through levels */ 1210 1207 int iCookie; /* Cookie value to store */ 1211 1208 1212 1209 assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); 1213 1210 memset(&buf, 0, sizeof(Fts5Buffer)); ................................................................................ 1232 1229 fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); 1233 1230 fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].nHeight); 1234 1231 fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); 1235 1232 fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); 1236 1233 } 1237 1234 } 1238 1235 1239 - fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n); 1236 + fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); 1240 1237 fts5BufferFree(&buf); 1241 1238 } 1242 1239 } 1243 1240 1244 1241 #if 0 1245 1242 static void fts5DebugStructure(int*,Fts5Buffer*,Fts5Structure*); 1246 1243 static void fts5PrintStructure(const char *zCaption, Fts5Structure *pStruct){ ................................................................................ 1528 1525 1529 1526 return pIter->bEof; 1530 1527 } 1531 1528 1532 1529 static Fts5DlidxIter *fts5DlidxIterInit( 1533 1530 Fts5Index *p, /* Fts5 Backend to iterate within */ 1534 1531 int bRev, /* True for ORDER BY ASC */ 1535 - int iIdx, int iSegid, /* Segment iSegid within index iIdx */ 1532 + int iSegid, /* Segment id */ 1536 1533 int iLeafPg /* Leaf page number to load dlidx for */ 1537 1534 ){ 1538 1535 Fts5DlidxIter *pIter; 1539 1536 1540 1537 pIter = (Fts5DlidxIter*)fts5IdxMalloc(p, sizeof(Fts5DlidxIter)); 1541 1538 if( pIter==0 ) return 0; 1542 1539 1543 - pIter->pData = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iIdx, iSegid, iLeafPg)); 1540 + pIter->pData = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iSegid, iLeafPg)); 1544 1541 if( pIter->pData==0 ){ 1545 1542 sqlite3_free(pIter); 1546 1543 pIter = 0; 1547 1544 }else{ 1548 1545 pIter->iLeafPgno = iLeafPg; 1549 1546 if( bRev==0 ){ 1550 1547 fts5DlidxIterFirst(pIter); ................................................................................ 1579 1576 Fts5SegIter *pIter /* Iterator to advance to next page */ 1580 1577 ){ 1581 1578 Fts5StructureSegment *pSeg = pIter->pSeg; 1582 1579 fts5DataRelease(pIter->pLeaf); 1583 1580 pIter->iLeafPgno++; 1584 1581 if( pIter->iLeafPgno<=pSeg->pgnoLast ){ 1585 1582 pIter->pLeaf = fts5DataRead(p, 1586 - FTS5_SEGMENT_ROWID(pIter->iIdx, pSeg->iSegid, 0, pIter->iLeafPgno) 1583 + FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pIter->iLeafPgno) 1587 1584 ); 1588 1585 }else{ 1589 1586 pIter->pLeaf = 0; 1590 1587 } 1591 1588 } 1592 1589 1593 1590 /* ................................................................................ 1665 1662 } 1666 1663 iOff += sqlite3GetVarint(&a[iOff], (u64*)&pIter->iRowid); 1667 1664 pIter->iLeafOffset = iOff; 1668 1665 } 1669 1666 1670 1667 /* 1671 1668 ** Initialize the iterator object pIter to iterate through the entries in 1672 -** segment pSeg within index iIdx. The iterator is left pointing to the 1673 -** first entry when this function returns. 1669 +** segment pSeg. The iterator is left pointing to the first entry when 1670 +** this function returns. 1674 1671 ** 1675 1672 ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 1676 1673 ** an error has already occurred when this function is called, it is a no-op. 1677 1674 */ 1678 1675 static void fts5SegIterInit( 1679 - Fts5Index *p, 1680 - int iIdx, /* Config.aHash[] index of FTS index */ 1676 + Fts5Index *p, /* FTS index object */ 1681 1677 Fts5StructureSegment *pSeg, /* Description of segment */ 1682 1678 Fts5SegIter *pIter /* Object to populate */ 1683 1679 ){ 1684 1680 if( pSeg->pgnoFirst==0 ){ 1685 1681 /* This happens if the segment is being used as an input to an incremental 1686 1682 ** merge and all data has already been "trimmed". See function 1687 1683 ** fts5TrimSegments() for details. In this case leave the iterator empty. ................................................................................ 1690 1686 assert( pIter->pLeaf==0 ); 1691 1687 return; 1692 1688 } 1693 1689 1694 1690 if( p->rc==SQLITE_OK ){ 1695 1691 memset(pIter, 0, sizeof(*pIter)); 1696 1692 pIter->pSeg = pSeg; 1697 - pIter->iIdx = iIdx; 1698 1693 pIter->iLeafPgno = pSeg->pgnoFirst-1; 1699 1694 fts5SegIterNextPage(p, pIter); 1700 1695 } 1701 1696 1702 1697 if( p->rc==SQLITE_OK ){ 1703 1698 u8 *a = pIter->pLeaf->p; 1704 1699 pIter->iLeafOffset = fts5GetU16(&a[2]); ................................................................................ 1767 1762 1768 1763 fts5DataRelease(pIter->pLeaf); 1769 1764 pIter->pLeaf = 0; 1770 1765 while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ 1771 1766 Fts5Data *pNew; 1772 1767 pIter->iLeafPgno--; 1773 1768 pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( 1774 - pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno 1769 + pIter->pSeg->iSegid, 0, pIter->iLeafPgno 1775 1770 )); 1776 1771 if( pNew ){ 1777 1772 if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ 1778 1773 if( pIter->iTermLeafOffset<pNew->n ){ 1779 1774 pIter->pLeaf = pNew; 1780 1775 pIter->iLeafOffset = pIter->iTermLeafOffset; 1781 1776 } ................................................................................ 1875 1870 pIter->iRowid += iDelta; 1876 1871 } 1877 1872 }else if( pIter->pSeg==0 ){ 1878 1873 const u8 *pList = 0; 1879 1874 const char *zTerm; 1880 1875 int nList; 1881 1876 if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ 1882 - sqlite3Fts5HashScanNext(p->apHash[0]); 1883 - sqlite3Fts5HashScanEntry(p->apHash[0], &zTerm, &pList, &nList); 1877 + sqlite3Fts5HashScanNext(p->pHash); 1878 + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); 1884 1879 } 1885 1880 if( pList==0 ){ 1886 1881 fts5DataRelease(pIter->pLeaf); 1887 1882 pIter->pLeaf = 0; 1888 1883 }else{ 1889 1884 pIter->pLeaf->p = (u8*)pList; 1890 1885 pIter->pLeaf->n = nList; ................................................................................ 1931 1926 #define SWAPVAL(T, a, b) { T tmp; tmp=a; a=b; b=tmp; } 1932 1927 1933 1928 /* 1934 1929 ** Iterator pIter currently points to the first rowid in a doclist. This 1935 1930 ** function sets the iterator up so that iterates in reverse order through 1936 1931 ** the doclist. 1937 1932 */ 1938 -static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){ 1933 +static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ 1939 1934 Fts5DlidxIter *pDlidx = pIter->pDlidx; 1940 1935 Fts5Data *pLast = 0; 1941 1936 int pgnoLast = 0; 1942 1937 1943 1938 if( pDlidx ){ 1944 1939 /* If the doclist-iterator is already at EOF, then the current doclist 1945 1940 ** contains no entries except those on the current page. */ 1946 1941 if( fts5DlidxIterEof(p, pDlidx)==0 ){ 1947 1942 int iSegid = pIter->pSeg->iSegid; 1948 1943 pgnoLast = pDlidx->iLeafPgno; 1949 - pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pgnoLast)); 1944 + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast)); 1950 1945 }else{ 1951 1946 pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); 1952 1947 } 1953 1948 }else{ 1954 1949 int iOff; /* Byte offset within pLeaf */ 1955 1950 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ 1956 1951 ................................................................................ 1985 1980 if( iOff>=pLeaf->n ){ 1986 1981 int pgno; 1987 1982 Fts5StructureSegment *pSeg = pIter->pSeg; 1988 1983 1989 1984 /* The last rowid in the doclist may not be on the current page. Search 1990 1985 ** forward to find the page containing the last rowid. */ 1991 1986 for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ 1992 - i64 iAbs = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pgno); 1987 + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pgno); 1993 1988 Fts5Data *pNew = fts5DataRead(p, iAbs); 1994 1989 if( pNew ){ 1995 1990 int iRowid, iTerm; 1996 1991 fts5LeafHeader(pNew, &iRowid, &iTerm); 1997 1992 if( iRowid ){ 1998 1993 SWAPVAL(Fts5Data*, pNew, pLast); 1999 1994 pgnoLast = pgno; ................................................................................ 2025 2020 pIter->iLeafOffset = iOff; 2026 2021 } 2027 2022 2028 2023 fts5SegIterReverseInitPage(p, pIter); 2029 2024 } 2030 2025 2031 2026 /* 2032 -** Iterator pIter currently points to the first rowid of a doclist within 2033 -** index iIdx. There is a doclist-index associated with the final term on 2034 -** the current page. If the current term is the last term on the page, 2035 -** load the doclist-index from disk and initialize an iterator at 2036 -** (pIter->pDlidx). 2027 +** Iterator pIter currently points to the first rowid of a doclist. 2028 +** There is a doclist-index associated with the final term on the current 2029 +** page. If the current term is the last term on the page, load the 2030 +** doclist-index from disk and initialize an iterator at (pIter->pDlidx). 2037 2031 */ 2038 -static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){ 2032 +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ 2039 2033 int iSeg = pIter->pSeg->iSegid; 2040 2034 int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); 2041 2035 Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ 2042 2036 2043 2037 assert( pIter->flags & FTS5_SEGITER_ONETERM ); 2044 2038 assert( pIter->pDlidx==0 ); 2045 2039 ................................................................................ 2058 2052 if( iDelta==0 ) return; 2059 2053 assert_nc( iOff<pLeaf->n ); 2060 2054 iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); 2061 2055 iOff += nPos; 2062 2056 } 2063 2057 } 2064 2058 2065 - pIter->pDlidx = fts5DlidxIterInit(p, bRev, iIdx, iSeg, pIter->iTermLeafPgno); 2059 + pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); 2066 2060 } 2067 2061 2068 2062 /* 2069 2063 ** Initialize the object pIter to point to term pTerm/nTerm within segment 2070 -** pSeg, index iIdx. If there is no such term in the index, the iterator 2071 -** is set to EOF. 2064 +** pSeg. If there is no such term in the index, the iterator is set to EOF. 2072 2065 ** 2073 2066 ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 2074 2067 ** an error has already occurred when this function is called, it is a no-op. 2075 2068 */ 2076 2069 static void fts5SegIterSeekInit( 2077 2070 Fts5Index *p, /* FTS5 backend */ 2078 - int iIdx, /* Config.aHash[] index of FTS index */ 2079 2071 const u8 *pTerm, int nTerm, /* Term to seek to */ 2080 2072 int flags, /* Mask of FTS5INDEX_XXX flags */ 2081 2073 Fts5StructureSegment *pSeg, /* Description of segment */ 2082 2074 Fts5SegIter *pIter /* Object to populate */ 2083 2075 ){ 2084 2076 int iPg = 1; 2085 2077 int h; 2086 - int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0); 2078 + int bGe = (flags & FTS5INDEX_QUERY_PREFIX); 2087 2079 int bDlidx = 0; /* True if there is a doclist-index */ 2088 2080 2089 2081 assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); 2090 2082 assert( pTerm && nTerm ); 2091 2083 memset(pIter, 0, sizeof(*pIter)); 2092 2084 pIter->pSeg = pSeg; 2093 - pIter->iIdx = iIdx; 2094 2085 2095 2086 /* This block sets stack variable iPg to the leaf page number that may 2096 2087 ** contain term (pTerm/nTerm), if it is present in the segment. */ 2097 2088 for(h=pSeg->nHeight-1; h>0; h--){ 2098 2089 Fts5NodeIter node; /* For iterating through internal nodes */ 2099 - i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, h, iPg); 2090 + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg); 2100 2091 Fts5Data *pNode = fts5DataRead(p, iRowid); 2101 2092 if( pNode==0 ) break; 2102 2093 2103 2094 fts5NodeIterInit(pNode->p, pNode->n, &node); 2104 2095 assert( node.term.n==0 ); 2105 2096 2106 2097 iPg = node.iChild; ................................................................................ 2145 2136 if( p->rc==SQLITE_OK && bGe==0 ){ 2146 2137 pIter->flags |= FTS5_SEGITER_ONETERM; 2147 2138 if( pIter->pLeaf ){ 2148 2139 if( flags & FTS5INDEX_QUERY_DESC ){ 2149 2140 pIter->flags |= FTS5_SEGITER_REVERSE; 2150 2141 } 2151 2142 if( bDlidx ){ 2152 - fts5SegIterLoadDlidx(p, iIdx, pIter); 2143 + fts5SegIterLoadDlidx(p, pIter); 2153 2144 } 2154 2145 if( flags & FTS5INDEX_QUERY_DESC ){ 2155 - fts5SegIterReverse(p, iIdx, pIter); 2146 + fts5SegIterReverse(p, pIter); 2156 2147 } 2157 2148 } 2158 2149 } 2159 2150 } 2160 2151 2161 2152 /* 2162 2153 ** Initialize the object pIter to point to term pTerm/nTerm within the 2163 -** in-memory hash table iIdx. If there is no such term in the table, the 2154 +** in-memory hash table. If there is no such term in the hash-table, the 2164 2155 ** iterator is set to EOF. 2165 2156 ** 2166 2157 ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 2167 2158 ** an error has already occurred when this function is called, it is a no-op. 2168 2159 */ 2169 2160 static void fts5SegIterHashInit( 2170 2161 Fts5Index *p, /* FTS5 backend */ 2171 - int iIdx, /* Config.aHash[] index of FTS index */ 2172 2162 const u8 *pTerm, int nTerm, /* Term to seek to */ 2173 2163 int flags, /* Mask of FTS5INDEX_XXX flags */ 2174 2164 Fts5SegIter *pIter /* Object to populate */ 2175 2165 ){ 2176 - Fts5Hash *pHash = p->apHash[iIdx]; 2177 2166 const u8 *pList = 0; 2178 2167 int nList = 0; 2179 2168 const u8 *z = 0; 2180 2169 int n = 0; 2181 2170 2182 - assert( pHash ); 2171 + assert( p->pHash ); 2183 2172 assert( p->rc==SQLITE_OK ); 2184 2173 2185 - if( pTerm==0 || (iIdx==0 && (flags & FTS5INDEX_QUERY_PREFIX)) ){ 2186 - p->rc = sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm); 2187 - sqlite3Fts5HashScanEntry(pHash, (const char**)&z, &pList, &nList); 2174 + if( pTerm==0 || (flags & FTS5INDEX_QUERY_PREFIX) ){ 2175 + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); 2176 + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); 2188 2177 n = (z ? strlen((const char*)z) : 0); 2189 2178 }else{ 2190 2179 pIter->flags |= FTS5_SEGITER_ONETERM; 2191 - sqlite3Fts5HashQuery(pHash, (const char*)pTerm, nTerm, &pList, &nList); 2180 + sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); 2192 2181 z = pTerm; 2193 2182 n = nTerm; 2194 2183 } 2195 2184 2196 2185 if( pList ){ 2197 2186 Fts5Data *pLeaf; 2198 2187 sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); ................................................................................ 2548 2537 ** 2549 2538 ** The iterator initially points to the first term/rowid entry in the 2550 2539 ** iterated data. 2551 2540 */ 2552 2541 static void fts5MultiIterNew( 2553 2542 Fts5Index *p, /* FTS5 backend to iterate within */ 2554 2543 Fts5Structure *pStruct, /* Structure of specific index */ 2555 - int iIdx, /* Config.aHash[] index of FTS index */ 2556 2544 int bSkipEmpty, /* True to ignore delete-keys */ 2557 2545 int flags, /* FTS5INDEX_QUERY_XXX flags */ 2558 2546 const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ 2559 2547 int iLevel, /* Level to iterate (-1 for all) */ 2560 2548 int nSegment, /* Number of segments to merge (iLevel>=0) */ 2561 2549 Fts5MultiSegIter **ppOut /* New object */ 2562 2550 ){ ................................................................................ 2570 2558 assert( (pTerm==0 && nTerm==0) || iLevel<0 ); 2571 2559 2572 2560 /* Allocate space for the new multi-seg-iterator. */ 2573 2561 if( p->rc==SQLITE_OK ){ 2574 2562 if( iLevel<0 ){ 2575 2563 assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); 2576 2564 nSeg = pStruct->nSegment; 2577 - nSeg += (p->apHash ? 1 : 0); 2565 + nSeg += (p->pHash ? 1 : 0); 2578 2566 }else{ 2579 2567 nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); 2580 2568 } 2581 2569 for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2); 2582 2570 } 2583 2571 2584 2572 *ppOut = pNew = fts5IdxMalloc(p, ................................................................................ 2592 2580 pNew->aFirst = (Fts5CResult*)&pNew->aSeg[nSlot]; 2593 2581 pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); 2594 2582 pNew->bSkipEmpty = bSkipEmpty; 2595 2583 2596 2584 /* Initialize each of the component segment iterators. */ 2597 2585 if( iLevel<0 ){ 2598 2586 Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; 2599 - if( p->apHash ){ 2587 + if( p->pHash ){ 2600 2588 /* Add a segment iterator for the current contents of the hash table. */ 2601 2589 Fts5SegIter *pIter = &pNew->aSeg[iIter++]; 2602 - fts5SegIterHashInit(p, iIdx, pTerm, nTerm, flags, pIter); 2590 + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); 2603 2591 } 2604 2592 for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){ 2605 2593 for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){ 2606 2594 Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; 2607 2595 Fts5SegIter *pIter = &pNew->aSeg[iIter++]; 2608 2596 if( pTerm==0 ){ 2609 - fts5SegIterInit(p, iIdx, pSeg, pIter); 2597 + fts5SegIterInit(p, pSeg, pIter); 2610 2598 }else{ 2611 - fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, flags, pSeg, pIter); 2599 + fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); 2612 2600 } 2613 2601 } 2614 2602 } 2615 2603 }else{ 2616 2604 pLvl = &pStruct->aLevel[iLevel]; 2617 2605 for(iSeg=nSeg-1; iSeg>=0; iSeg--){ 2618 - fts5SegIterInit(p, iIdx, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); 2606 + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); 2619 2607 } 2620 2608 } 2621 2609 assert( iIter==nSeg ); 2622 2610 2623 2611 /* If the above was successful, each component iterators now points 2624 2612 ** to the first entry in its segment. In this case initialize the 2625 2613 ** aFirst[] array. Or, if an error has occurred, free the iterator ................................................................................ 2730 2718 int iOff = pSeg->iLeafOffset; 2731 2719 2732 2720 memset(pIter, 0, sizeof(*pIter)); 2733 2721 /* If Fts5SegIter.pSeg is NULL, then this iterator iterates through data 2734 2722 ** currently stored in a hash table. In this case there is no leaf-rowid 2735 2723 ** to calculate. */ 2736 2724 if( pSeg->pSeg ){ 2737 - int iId = pSeg->pSeg->iSegid; 2738 - i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno); 2725 + i64 rowid = FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, 0, pSeg->iLeafPgno); 2739 2726 pIter->iLeafRowid = rowid; 2740 2727 } 2741 2728 2742 2729 fts5DataReference(pLeaf); 2743 2730 pIter->pLeaf = pLeaf; 2744 2731 pIter->nRem = pSeg->nPos; 2745 2732 pIter->n = MIN(pLeaf->n - iOff, pIter->nRem); ................................................................................ 2790 2777 return (int)iSegid; 2791 2778 } 2792 2779 2793 2780 /* 2794 2781 ** Discard all data currently cached in the hash-tables. 2795 2782 */ 2796 2783 static void fts5IndexDiscardData(Fts5Index *p){ 2797 - assert( p->apHash || p->nPendingData==0 ); 2798 - if( p->apHash ){ 2799 - Fts5Config *pConfig = p->pConfig; 2800 - int i; 2801 - for(i=0; i<=pConfig->nPrefix; i++){ 2802 - sqlite3Fts5HashClear(p->apHash[i]); 2803 - } 2784 + assert( p->pHash || p->nPendingData==0 ); 2785 + if( p->pHash ){ 2786 + sqlite3Fts5HashClear(p->pHash); 2804 2787 p->nPendingData = 0; 2805 2788 } 2806 2789 } 2807 2790 2808 2791 /* 2809 2792 ** Return the size of the prefix, in bytes, that buffer (nNew/pNew) shares 2810 2793 ** with buffer (nOld/pOld). ................................................................................ 2828 2811 static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){ 2829 2812 if( pWriter->nEmpty ){ 2830 2813 int bFlag = 0; 2831 2814 Fts5PageWriter *pPg; 2832 2815 pPg = &pWriter->aWriter[1]; 2833 2816 if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ 2834 2817 i64 iKey = FTS5_DOCLIST_IDX_ROWID( 2835 - pWriter->iIdx, pWriter->iSegid, 2836 - pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty 2818 + pWriter->iSegid, pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty 2837 2819 ); 2838 2820 assert( pWriter->cdlidx.n>0 ); 2839 2821 fts5DataWrite(p, iKey, pWriter->cdlidx.p, pWriter->cdlidx.n); 2840 2822 bFlag = 1; 2841 2823 } 2842 2824 fts5BufferAppendVarint(&p->rc, &pPg->buf, bFlag); 2843 2825 fts5BufferAppendVarint(&p->rc, &pPg->buf, pWriter->nEmpty); ................................................................................ 2897 2879 pPage = &pWriter->aWriter[iHeight]; 2898 2880 2899 2881 fts5WriteBtreeNEmpty(p, pWriter); 2900 2882 2901 2883 if( pPage->buf.n>=p->pConfig->pgsz ){ 2902 2884 /* pPage will be written to disk. The term will be written into the 2903 2885 ** parent of pPage. */ 2904 - i64 iRowid = FTS5_SEGMENT_ROWID( 2905 - pWriter->iIdx, pWriter->iSegid, iHeight, pPage->pgno 2906 - ); 2886 + i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno); 2907 2887 fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); 2908 2888 fts5BufferZero(&pPage->buf); 2909 2889 fts5BufferZero(&pPage->term); 2910 2890 fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno); 2911 2891 pPage->pgno++; 2912 2892 }else{ 2913 2893 int nPre = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); ................................................................................ 2967 2947 if( pWriter->bFirstTermInPage ){ 2968 2948 /* No term was written to this page. */ 2969 2949 assert( 0==fts5GetU16(&pPage->buf.p[2]) ); 2970 2950 fts5WriteBtreeNoTerm(p, pWriter); 2971 2951 } 2972 2952 2973 2953 /* Write the current page to the db. */ 2974 - iRowid = FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, 0, pPage->pgno); 2954 + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, 0, pPage->pgno); 2975 2955 fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); 2976 2956 2977 2957 /* Initialize the next page. */ 2978 2958 fts5BufferZero(&pPage->buf); 2979 2959 fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); 2980 2960 pPage->pgno++; 2981 2961 ................................................................................ 3175 3155 fts5WriteBtreeNEmpty(p, pWriter); 3176 3156 } 3177 3157 *pnHeight = pWriter->nWriter; 3178 3158 3179 3159 for(i=1; i<pWriter->nWriter; i++){ 3180 3160 Fts5PageWriter *pPg = &pWriter->aWriter[i]; 3181 3161 fts5DataWrite(p, 3182 - FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pPg->pgno), 3162 + FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno), 3183 3163 pPg->buf.p, pPg->buf.n 3184 3164 ); 3185 3165 } 3186 3166 } 3187 3167 } 3188 3168 for(i=0; i<pWriter->nWriter; i++){ 3189 3169 Fts5PageWriter *pPg = &pWriter->aWriter[i]; ................................................................................ 3193 3173 sqlite3_free(pWriter->aWriter); 3194 3174 sqlite3Fts5BufferFree(&pWriter->cdlidx); 3195 3175 } 3196 3176 3197 3177 static void fts5WriteInit( 3198 3178 Fts5Index *p, 3199 3179 Fts5SegWriter *pWriter, 3200 - int iIdx, int iSegid 3180 + int iSegid 3201 3181 ){ 3202 3182 memset(pWriter, 0, sizeof(Fts5SegWriter)); 3203 - pWriter->iIdx = iIdx; 3204 3183 pWriter->iSegid = iSegid; 3205 3184 3206 3185 pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p,sizeof(Fts5PageWriter)); 3207 3186 if( pWriter->aWriter==0 ) return; 3208 3187 pWriter->nWriter = 1; 3209 3188 pWriter->aWriter[0].pgno = 1; 3210 3189 pWriter->bFirstTermInPage = 1; 3211 3190 } 3212 3191 3213 3192 static void fts5WriteInitForAppend( 3214 3193 Fts5Index *p, /* FTS5 backend object */ 3215 3194 Fts5SegWriter *pWriter, /* Writer to initialize */ 3216 - int iIdx, /* Index segment is a part of */ 3217 3195 Fts5StructureSegment *pSeg /* Segment object to append to */ 3218 3196 ){ 3219 3197 int nByte = pSeg->nHeight * sizeof(Fts5PageWriter); 3220 3198 memset(pWriter, 0, sizeof(Fts5SegWriter)); 3221 - pWriter->iIdx = iIdx; 3222 3199 pWriter->iSegid = pSeg->iSegid; 3223 3200 pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte); 3224 3201 3225 3202 if( p->rc==SQLITE_OK ){ 3226 3203 int pgno = 1; 3227 3204 int i; 3228 3205 pWriter->nWriter = pSeg->nHeight; 3229 3206 pWriter->aWriter[0].pgno = pSeg->pgnoLast+1; 3230 3207 for(i=pSeg->nHeight-1; i>0; i--){ 3231 - i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pgno); 3208 + i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno); 3232 3209 Fts5PageWriter *pPg = &pWriter->aWriter[i]; 3233 3210 pPg->pgno = pgno; 3234 3211 fts5DataBuffer(p, &pPg->buf, iRowid); 3235 3212 if( p->rc==SQLITE_OK ){ 3236 3213 Fts5NodeIter ss; 3237 3214 fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss); 3238 3215 while( ss.aData ) fts5NodeIterNext(&p->rc, &ss); ................................................................................ 3272 3249 }else{ 3273 3250 int iOff = pSeg->iTermLeafOffset; /* Offset on new first leaf page */ 3274 3251 i64 iLeafRowid; 3275 3252 Fts5Data *pData; 3276 3253 int iId = pSeg->pSeg->iSegid; 3277 3254 u8 aHdr[4] = {0x00, 0x00, 0x00, 0x04}; 3278 3255 3279 - iLeafRowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iTermLeafPgno); 3256 + iLeafRowid = FTS5_SEGMENT_ROWID(iId, 0, pSeg->iTermLeafPgno); 3280 3257 pData = fts5DataRead(p, iLeafRowid); 3281 3258 if( pData ){ 3282 3259 fts5BufferZero(&buf); 3283 3260 fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); 3284 3261 fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); 3285 3262 fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); 3286 3263 fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); 3287 3264 fts5DataRelease(pData); 3288 3265 pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; 3289 - fts5DataDelete(p, FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, 1),iLeafRowid); 3266 + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid); 3290 3267 fts5DataWrite(p, iLeafRowid, buf.p, buf.n); 3291 3268 } 3292 3269 } 3293 3270 } 3294 3271 fts5BufferFree(&buf); 3295 3272 } 3296 3273 3297 3274 /* 3298 3275 ** 3299 3276 */ 3300 3277 static void fts5IndexMergeLevel( 3301 3278 Fts5Index *p, /* FTS5 backend object */ 3302 - int iIdx, /* Index to work on */ 3303 - Fts5Structure **ppStruct, /* IN/OUT: Stucture of index iIdx */ 3279 + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ 3304 3280 int iLvl, /* Level to read input from */ 3305 3281 int *pnRem /* Write up to this many output leaves */ 3306 3282 ){ 3307 3283 Fts5Structure *pStruct = *ppStruct; 3308 3284 Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; 3309 3285 Fts5StructureLevel *pLvlOut; 3310 3286 Fts5MultiSegIter *pIter = 0; /* Iterator to read input data */ ................................................................................ 3317 3293 int bOldest; /* True if the output segment is the oldest */ 3318 3294 3319 3295 assert( iLvl<pStruct->nLevel ); 3320 3296 assert( pLvl->nMerge<=pLvl->nSeg ); 3321 3297 3322 3298 memset(&writer, 0, sizeof(Fts5SegWriter)); 3323 3299 memset(&term, 0, sizeof(Fts5Buffer)); 3324 - writer.iIdx = iIdx; 3325 3300 if( pLvl->nMerge ){ 3326 3301 pLvlOut = &pStruct->aLevel[iLvl+1]; 3327 3302 assert( pLvlOut->nSeg>0 ); 3328 3303 nInput = pLvl->nMerge; 3329 - fts5WriteInitForAppend(p, &writer, iIdx, &pLvlOut->aSeg[pLvlOut->nSeg-1]); 3304 + fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]); 3330 3305 pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; 3331 3306 }else{ 3332 3307 int iSegid = fts5AllocateSegid(p, pStruct); 3333 3308 3334 3309 /* Extend the Fts5Structure object as required to ensure the output 3335 3310 ** segment exists. */ 3336 3311 if( iLvl==pStruct->nLevel-1 ){ ................................................................................ 3338 3313 pStruct = *ppStruct; 3339 3314 } 3340 3315 fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); 3341 3316 if( p->rc ) return; 3342 3317 pLvl = &pStruct->aLevel[iLvl]; 3343 3318 pLvlOut = &pStruct->aLevel[iLvl+1]; 3344 3319 3345 - fts5WriteInit(p, &writer, iIdx, iSegid); 3320 + fts5WriteInit(p, &writer, iSegid); 3346 3321 3347 3322 /* Add the new segment to the output level */ 3348 3323 pSeg = &pLvlOut->aSeg[pLvlOut->nSeg]; 3349 3324 pLvlOut->nSeg++; 3350 3325 pSeg->pgnoFirst = 1; 3351 3326 pSeg->iSegid = iSegid; 3352 3327 pStruct->nSegment++; ................................................................................ 3358 3333 3359 3334 #if 0 3360 3335 fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl); 3361 3336 fflush(stdout); 3362 3337 #endif 3363 3338 3364 3339 assert( iLvl>=0 ); 3365 - for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter); 3340 + for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter); 3366 3341 fts5MultiIterEof(p, pIter)==0; 3367 3342 fts5MultiIterNext(p, pIter, 0, 0) 3368 3343 ){ 3369 3344 Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; 3370 3345 Fts5ChunkIter sPos; /* Used to iterate through position list */ 3371 3346 int nPos; /* position-list size field value */ 3372 3347 int nTerm; ................................................................................ 3410 3385 fts5WriteFinish(p, &writer, &pSeg->nHeight, &pSeg->pgnoLast); 3411 3386 3412 3387 if( fts5MultiIterEof(p, pIter) ){ 3413 3388 int i; 3414 3389 3415 3390 /* Remove the redundant segments from the %_data table */ 3416 3391 for(i=0; i<nInput; i++){ 3417 - fts5DataRemoveSegment(p, iIdx, pLvl->aSeg[i].iSegid); 3392 + fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid); 3418 3393 } 3419 3394 3420 3395 /* Remove the redundant segments from the input level */ 3421 3396 if( pLvl->nSeg!=nInput ){ 3422 3397 int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); 3423 3398 memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove); 3424 3399 } ................................................................................ 3437 3412 3438 3413 fts5MultiIterFree(p, pIter); 3439 3414 fts5BufferFree(&term); 3440 3415 if( pnRem ) *pnRem -= writer.nLeafWritten; 3441 3416 } 3442 3417 3443 3418 /* 3444 -** Do up to nPg pages of automerge work on index iIdx. 3419 +** Do up to nPg pages of automerge work on the index. 3445 3420 */ 3446 3421 static void fts5IndexMerge( 3447 3422 Fts5Index *p, /* FTS5 backend object */ 3448 - int iIdx, /* Index to work on */ 3449 3423 Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ 3450 3424 int nPg /* Pages of work to do */ 3451 3425 ){ 3452 3426 int nRem = nPg; 3453 3427 Fts5Structure *pStruct = *ppStruct; 3454 3428 while( nRem>0 && p->rc==SQLITE_OK ){ 3455 3429 int iLvl; /* To iterate through levels */ ................................................................................ 3481 3455 #endif 3482 3456 3483 3457 if( nBest<p->pConfig->nAutomerge 3484 3458 && pStruct->aLevel[iBestLvl].nMerge==0 3485 3459 ){ 3486 3460 break; 3487 3461 } 3488 - fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem); 3462 + fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); 3489 3463 if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ 3490 3464 fts5StructurePromote(p, iBestLvl+1, pStruct); 3491 3465 } 3492 3466 } 3493 3467 *ppStruct = pStruct; 3494 3468 } 3495 3469 3496 3470 /* 3497 3471 ** A total of nLeaf leaf pages of data has just been flushed to a level-0 3498 -** segments in index iIdx with structure pStruct. This function updates the 3499 -** write-counter accordingly and, if necessary, performs incremental merge 3500 -** work. 3472 +** segment. This function updates the write-counter accordingly and, if 3473 +** necessary, performs incremental merge work. 3501 3474 ** 3502 3475 ** If an error occurs, set the Fts5Index.rc error code. If an error has 3503 3476 ** already occurred, this function is a no-op. 3504 3477 */ 3505 3478 static void fts5IndexAutomerge( 3506 3479 Fts5Index *p, /* FTS5 backend object */ 3507 - int iIdx, /* Index to work on */ 3508 3480 Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ 3509 3481 int nLeaf /* Number of output leaves just written */ 3510 3482 ){ 3511 3483 if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){ 3512 3484 Fts5Structure *pStruct = *ppStruct; 3513 3485 i64 nWrite; /* Initial value of write-counter */ 3514 3486 int nWork; /* Number of work-quanta to perform */ ................................................................................ 3516 3488 3517 3489 /* Update the write-counter. While doing so, set nWork. */ 3518 3490 nWrite = pStruct->nWriteCounter; 3519 3491 nWork = ((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit); 3520 3492 pStruct->nWriteCounter += nLeaf; 3521 3493 nRem = p->nWorkUnit * nWork * pStruct->nLevel; 3522 3494 3523 - fts5IndexMerge(p, iIdx, ppStruct, nRem); 3495 + fts5IndexMerge(p, ppStruct, nRem); 3524 3496 } 3525 3497 } 3526 3498 3527 3499 static void fts5IndexCrisismerge( 3528 3500 Fts5Index *p, /* FTS5 backend object */ 3529 - int iIdx, /* Index to work on */ 3530 3501 Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ 3531 3502 ){ 3532 3503 const int nCrisis = p->pConfig->nCrisisMerge; 3533 3504 Fts5Structure *pStruct = *ppStruct; 3534 3505 int iLvl = 0; 3535 3506 3536 3507 assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); 3537 3508 while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ 3538 - fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0); 3509 + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); 3539 3510 fts5StructurePromote(p, iLvl+1, pStruct); 3540 3511 iLvl++; 3541 3512 } 3542 3513 *ppStruct = pStruct; 3543 3514 } 3544 3515 3545 3516 static int fts5IndexReturn(Fts5Index *p){ ................................................................................ 3580 3551 /* 3581 3552 ** Flush the contents of in-memory hash table iHash to a new level-0 3582 3553 ** segment on disk. Also update the corresponding structure record. 3583 3554 ** 3584 3555 ** If an error occurs, set the Fts5Index.rc error code. If an error has 3585 3556 ** already occurred, this function is a no-op. 3586 3557 */ 3587 -static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){ 3588 - Fts5Hash *pHash = p->apHash[iHash]; 3558 +static void fts5FlushOneHash(Fts5Index *p){ 3559 + Fts5Hash *pHash = p->pHash; 3589 3560 Fts5Structure *pStruct; 3590 3561 int iSegid; 3591 3562 int pgnoLast = 0; /* Last leaf page number in segment */ 3592 3563 3593 3564 /* Obtain a reference to the index structure and allocate a new segment-id 3594 3565 ** for the new level-0 segment. */ 3595 - pStruct = fts5StructureRead(p, iHash); 3566 + pStruct = fts5StructureRead(p); 3596 3567 iSegid = fts5AllocateSegid(p, pStruct); 3597 3568 3598 3569 if( iSegid ){ 3599 3570 const int pgsz = p->pConfig->pgsz; 3600 3571 3601 3572 Fts5StructureSegment *pSeg; /* New segment within pStruct */ 3602 3573 int nHeight; /* Height of new segment b-tree */ 3603 3574 Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ 3604 3575 const u8 *zPrev = 0; 3605 3576 3606 3577 Fts5SegWriter writer; 3607 - fts5WriteInit(p, &writer, iHash, iSegid); 3578 + fts5WriteInit(p, &writer, iSegid); 3608 3579 3609 3580 /* Pre-allocate the buffer used to assemble leaf pages to the target 3610 3581 ** page size. */ 3611 3582 assert( pgsz>0 ); 3612 3583 pBuf = &writer.aWriter[0].buf; 3613 3584 fts5BufferGrow(&p->rc, pBuf, pgsz + 20); 3614 3585 ................................................................................ 3745 3716 pSeg->pgnoLast = pgnoLast; 3746 3717 pStruct->nSegment++; 3747 3718 } 3748 3719 fts5StructurePromote(p, 0, pStruct); 3749 3720 } 3750 3721 3751 3722 3752 - fts5IndexAutomerge(p, iHash, &pStruct, pgnoLast); 3753 - fts5IndexCrisismerge(p, iHash, &pStruct); 3754 - fts5StructureWrite(p, iHash, pStruct); 3723 + fts5IndexAutomerge(p, &pStruct, pgnoLast); 3724 + fts5IndexCrisismerge(p, &pStruct); 3725 + fts5StructureWrite(p, pStruct); 3755 3726 fts5StructureRelease(pStruct); 3756 3727 } 3757 3728 3758 3729 /* 3759 3730 ** Flush any data stored in the in-memory hash tables to the database. 3760 3731 */ 3761 3732 static void fts5IndexFlush(Fts5Index *p){ 3762 - Fts5Config *pConfig = p->pConfig; 3763 - int i; /* Used to iterate through indexes */ 3764 - int nLeaf = 0; /* Number of leaves written */ 3765 - 3766 - /* If an error has already occured this call is a no-op. */ 3767 - if( p->nPendingData==0 ) return; 3768 - assert( p->apHash ); 3769 - 3770 - /* Flush the terms and each prefix index to disk */ 3771 - for(i=0; i<=pConfig->nPrefix; i++){ 3772 - fts5FlushOneHash(p, i, &nLeaf); 3733 + /* Unless it is empty, flush the hash table to disk */ 3734 + if( p->rc==SQLITE_OK && p->nPendingData ){ 3735 + assert( p->pHash ); 3736 + p->nPendingData = 0; 3737 + fts5FlushOneHash(p); 3773 3738 } 3774 - p->nPendingData = 0; 3775 3739 } 3776 3740 3777 3741 3778 3742 int sqlite3Fts5IndexOptimize(Fts5Index *p){ 3779 - Fts5Config *pConfig = p->pConfig; 3780 - int i; 3743 + Fts5Structure *pStruct; 3744 + Fts5Structure *pNew = 0; 3745 + int nSeg = 0; 3781 3746 3782 3747 assert( p->rc==SQLITE_OK ); 3783 3748 fts5IndexFlush(p); 3784 - for(i=0; i<=pConfig->nPrefix; i++){ 3785 - Fts5Structure *pStruct = fts5StructureRead(p, i); 3786 - Fts5Structure *pNew = 0; 3787 - int nSeg = 0; 3788 - if( pStruct ){ 3789 - assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); 3790 - nSeg = pStruct->nSegment; 3791 - if( nSeg>1 ){ 3792 - int nByte = sizeof(Fts5Structure); 3793 - nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); 3794 - pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); 3795 - } 3796 - } 3797 - if( pNew ){ 3798 - Fts5StructureLevel *pLvl; 3799 - int nByte = nSeg * sizeof(Fts5StructureSegment); 3800 - pNew->nLevel = pStruct->nLevel+1; 3801 - pNew->nWriteCounter = pStruct->nWriteCounter; 3802 - pLvl = &pNew->aLevel[pStruct->nLevel]; 3803 - pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); 3804 - if( pLvl->aSeg ){ 3805 - int iLvl, iSeg; 3806 - int iSegOut = 0; 3807 - for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 3808 - for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 3809 - pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; 3810 - iSegOut++; 3811 - } 3749 + pStruct = fts5StructureRead(p); 3750 + 3751 + if( pStruct ){ 3752 + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); 3753 + nSeg = pStruct->nSegment; 3754 + if( nSeg>1 ){ 3755 + int nByte = sizeof(Fts5Structure); 3756 + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); 3757 + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); 3758 + } 3759 + } 3760 + if( pNew ){ 3761 + Fts5StructureLevel *pLvl; 3762 + int nByte = nSeg * sizeof(Fts5StructureSegment); 3763 + pNew->nLevel = pStruct->nLevel+1; 3764 + pNew->nWriteCounter = pStruct->nWriteCounter; 3765 + pLvl = &pNew->aLevel[pStruct->nLevel]; 3766 + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); 3767 + if( pLvl->aSeg ){ 3768 + int iLvl, iSeg; 3769 + int iSegOut = 0; 3770 + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 3771 + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 3772 + pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; 3773 + iSegOut++; 3812 3774 } 3813 - pNew->nSegment = pLvl->nSeg = nSeg; 3814 - }else{ 3815 - sqlite3_free(pNew); 3816 - pNew = 0; 3817 - } 3818 - } 3819 - 3820 - if( pNew ){ 3821 - int iLvl = pNew->nLevel-1; 3822 - while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ 3823 - int nRem = FTS5_OPT_WORK_UNIT; 3824 - fts5IndexMergeLevel(p, i, &pNew, iLvl, &nRem); 3825 - } 3826 - 3827 - fts5StructureWrite(p, i, pNew); 3828 - fts5StructureRelease(pNew); 3829 - } 3830 - 3831 - fts5StructureRelease(pStruct); 3832 - } 3833 - 3775 + } 3776 + pNew->nSegment = pLvl->nSeg = nSeg; 3777 + }else{ 3778 + sqlite3_free(pNew); 3779 + pNew = 0; 3780 + } 3781 + } 3782 + 3783 + if( pNew ){ 3784 + int iLvl = pNew->nLevel-1; 3785 + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ 3786 + int nRem = FTS5_OPT_WORK_UNIT; 3787 + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); 3788 + } 3789 + 3790 + fts5StructureWrite(p, pNew); 3791 + fts5StructureRelease(pNew); 3792 + } 3793 + 3794 + fts5StructureRelease(pStruct); 3834 3795 return fts5IndexReturn(p); 3835 3796 } 3836 3797 3837 3798 int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ 3838 3799 Fts5Structure *pStruct; 3839 3800 3840 - pStruct = fts5StructureRead(p, 0); 3841 - fts5IndexMerge(p, 0, &pStruct, nMerge); 3842 - fts5StructureWrite(p, 0, pStruct); 3801 + pStruct = fts5StructureRead(p); 3802 + fts5IndexMerge(p, &pStruct, nMerge); 3803 + fts5StructureWrite(p, pStruct); 3843 3804 fts5StructureRelease(pStruct); 3844 3805 3845 3806 return fts5IndexReturn(p); 3846 3807 } 3847 3808 3848 3809 3849 3810 /* ................................................................................ 4036 3997 Fts5IndexIter *pIter /* Populate this object */ 4037 3998 ){ 4038 3999 Fts5Structure *pStruct; 4039 4000 Fts5Buffer *aBuf; 4040 4001 const int nBuf = 32; 4041 4002 4042 4003 aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); 4043 - pStruct = fts5StructureRead(p, 0); 4004 + pStruct = fts5StructureRead(p); 4044 4005 4045 4006 if( aBuf && pStruct ){ 4046 4007 Fts5DoclistIter *pDoclist; 4047 4008 int i; 4048 4009 i64 iLastRowid = 0; 4049 4010 Fts5MultiSegIter *p1 = 0; /* Iterator used to gather data from index */ 4050 4011 Fts5Buffer doclist; 4051 4012 4052 4013 memset(&doclist, 0, sizeof(doclist)); 4053 - for(fts5MultiIterNew(p, pStruct, 0, 1, 1, pToken, nToken, -1, 0, &p1); 4014 + for(fts5MultiIterNew(p, pStruct, 1, 1, pToken, nToken, -1, 0, &p1); 4054 4015 fts5MultiIterEof(p, p1)==0; 4055 4016 fts5MultiIterNext(p, p1, 0, 0) 4056 4017 ){ 4057 4018 i64 iRowid = fts5MultiIterRowid(p1); 4058 4019 int nTerm; 4059 4020 const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); 4060 4021 assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); ................................................................................ 4109 4070 /* 4110 4071 ** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain 4111 4072 ** to the document with rowid iRowid. 4112 4073 */ 4113 4074 int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ 4114 4075 assert( p->rc==SQLITE_OK ); 4115 4076 4116 - /* Allocate hash tables if they have not already been allocated */ 4117 - if( p->apHash==0 ){ 4118 - int i; 4119 - int rc = SQLITE_OK; 4120 - int nHash = p->pConfig->nPrefix + 1; 4121 - Fts5Hash **apNew; 4122 - 4123 - apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash); 4124 - for(i=0; rc==SQLITE_OK && i<nHash; i++){ 4125 - rc = sqlite3Fts5HashNew(&apNew[i], &p->nPendingData); 4126 - } 4127 - if( rc==SQLITE_OK ){ 4128 - p->apHash = apNew; 4129 - }else{ 4130 - if( apNew ){ 4131 - for(i=0; i<nHash; i++){ 4132 - sqlite3Fts5HashFree(apNew[i]); 4133 - } 4134 - sqlite3_free(apNew); 4135 - } 4136 - return rc; 4137 - } 4077 + /* Allocate the hash table if it has not already been allocated */ 4078 + if( p->pHash==0 ){ 4079 + p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); 4138 4080 } 4139 4081 4140 4082 if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ 4141 - assert( p->rc==SQLITE_OK ); 4142 4083 fts5IndexFlush(p); 4143 4084 } 4144 4085 p->iWriteRowid = iRowid; 4145 4086 return fts5IndexReturn(p); 4146 4087 } 4147 4088 4148 4089 /* ................................................................................ 4170 4111 4171 4112 /* 4172 4113 ** The %_data table is completely empty when this function is called. This 4173 4114 ** function populates it with the initial structure objects for each index, 4174 4115 ** and the initial version of the "averages" record (a zero-byte blob). 4175 4116 */ 4176 4117 int sqlite3Fts5IndexReinit(Fts5Index *p){ 4177 - int i; 4178 4118 Fts5Structure s; 4179 4119 4120 + assert( p->rc==SQLITE_OK ); 4121 + p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); 4122 + 4180 4123 memset(&s, 0, sizeof(Fts5Structure)); 4181 - for(i=0; i<p->pConfig->nPrefix+1; i++){ 4182 - fts5StructureWrite(p, i, &s); 4183 - } 4184 - if( p->rc==SQLITE_OK ){ 4185 - p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); 4186 - } 4124 + fts5StructureWrite(p, &s); 4187 4125 4188 4126 return fts5IndexReturn(p); 4189 4127 } 4190 4128 4191 4129 /* 4192 4130 ** Open a new Fts5Index handle. If the bCreate argument is true, create 4193 4131 ** and initialize the underlying %_data table. ................................................................................ 4235 4173 */ 4236 4174 int sqlite3Fts5IndexClose(Fts5Index *p){ 4237 4175 int rc = SQLITE_OK; 4238 4176 if( p ){ 4239 4177 assert( p->pReader==0 ); 4240 4178 sqlite3_finalize(p->pWriter); 4241 4179 sqlite3_finalize(p->pDeleter); 4242 - if( p->apHash ){ 4243 - int i; 4244 - for(i=0; i<=p->pConfig->nPrefix; i++){ 4245 - sqlite3Fts5HashFree(p->apHash[i]); 4246 - } 4247 - sqlite3_free(p->apHash); 4248 - } 4180 + sqlite3Fts5HashFree(p->pHash); 4181 + sqlite3Fts5BufferFree(&p->scratch); 4249 4182 sqlite3_free(p->zDataTbl); 4250 4183 sqlite3_free(p); 4251 4184 } 4252 4185 return rc; 4253 4186 } 4254 4187 4255 4188 /* ................................................................................ 4298 4231 int sqlite3Fts5IndexWrite( 4299 4232 Fts5Index *p, /* Index to write to */ 4300 4233 int iCol, /* Column token appears in (-ve -> delete) */ 4301 4234 int iPos, /* Position of token within column */ 4302 4235 const char *pToken, int nToken /* Token to add or remove to or from index */ 4303 4236 ){ 4304 4237 int i; /* Used to iterate through indexes */ 4305 - int rc; /* Return code */ 4238 + int rc = SQLITE_OK; /* Return code */ 4306 4239 Fts5Config *pConfig = p->pConfig; 4307 4240 4308 4241 assert( p->rc==SQLITE_OK ); 4309 4242 4310 - /* Add the new token to the main terms hash table. And to each of the 4311 - ** prefix hash tables that it is large enough for. */ 4243 + /* Add the entry to the main terms index. */ 4312 4244 rc = sqlite3Fts5HashWrite( 4313 - p->apHash[0], p->iWriteRowid, iCol, iPos, pToken, nToken 4245 + p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken 4314 4246 ); 4247 + 4315 4248 for(i=0; i<pConfig->nPrefix && rc==SQLITE_OK; i++){ 4316 4249 int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); 4317 4250 if( nByte ){ 4318 - rc = sqlite3Fts5HashWrite( 4319 - p->apHash[i+1], p->iWriteRowid, iCol, iPos, pToken, nByte 4251 + rc = sqlite3Fts5HashWrite(p->pHash, 4252 + p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX+i+1, pToken, nByte 4320 4253 ); 4321 4254 } 4322 4255 } 4323 4256 4324 4257 return rc; 4325 4258 } 4326 4259 ................................................................................ 4333 4266 const char *pToken, int nToken, /* Token (or prefix) to query for */ 4334 4267 int flags, /* Mask of FTS5INDEX_QUERY_X flags */ 4335 4268 Fts5IndexIter **ppIter /* OUT: New iterator object */ 4336 4269 ){ 4337 4270 Fts5Config *pConfig = p->pConfig; 4338 4271 Fts5IndexIter *pRet; 4339 4272 int iIdx = 0; 4273 + Fts5Buffer buf = {0, 0, 0}; 4274 + 4275 + if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ 4276 + memcpy(&buf.p[1], pToken, nToken); 4277 + } 4340 4278 4341 4279 if( flags & FTS5INDEX_QUERY_PREFIX ){ 4342 4280 if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ 4343 4281 iIdx = 1+pConfig->nPrefix; 4344 4282 }else{ 4345 4283 int nChar = fts5IndexCharlen(pToken, nToken); 4346 4284 for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ ................................................................................ 4351 4289 4352 4290 pRet = (Fts5IndexIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5IndexIter)); 4353 4291 if( pRet ){ 4354 4292 memset(pRet, 0, sizeof(Fts5IndexIter)); 4355 4293 4356 4294 pRet->pIndex = p; 4357 4295 if( iIdx<=pConfig->nPrefix ){ 4358 - pRet->pStruct = fts5StructureRead(p, iIdx); 4296 + buf.p[0] = FTS5_MAIN_PREFIX + iIdx; 4297 + pRet->pStruct = fts5StructureRead(p); 4359 4298 if( pRet->pStruct ){ 4360 - fts5MultiIterNew(p, pRet->pStruct, 4361 - iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti 4299 + int f = (flags & ~FTS5INDEX_QUERY_PREFIX); 4300 + fts5MultiIterNew( 4301 + p, pRet->pStruct, 1, f, buf.p, nToken+1, -1, 0, &pRet->pMulti 4362 4302 ); 4363 4303 } 4364 4304 }else{ 4365 4305 int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; 4366 - fts5SetupPrefixIter(p, bDesc, (const u8*)pToken, nToken, pRet); 4306 + buf.p[0] = FTS5_MAIN_PREFIX; 4307 + fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pRet); 4367 4308 } 4368 4309 } 4369 4310 4370 4311 if( p->rc ){ 4371 4312 sqlite3Fts5IterClose(pRet); 4372 4313 pRet = 0; 4373 4314 } 4374 4315 *ppIter = pRet; 4316 + sqlite3Fts5BufferFree(&buf); 4375 4317 return fts5IndexReturn(p); 4376 4318 } 4377 4319 4378 4320 /* 4379 4321 ** Return true if the iterator passed as the only argument is at EOF. 4380 4322 */ 4381 4323 int sqlite3Fts5IterEof(Fts5IndexIter *pIter){ ................................................................................ 4516 4458 ** Set the 32-bit cookie value stored at the start of all structure 4517 4459 ** records to the value passed as the second argument. 4518 4460 ** 4519 4461 ** Return SQLITE_OK if successful, or an SQLite error code if an error 4520 4462 ** occurs. 4521 4463 */ 4522 4464 int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ 4523 - int rc = SQLITE_OK; 4524 - Fts5Config *pConfig = p->pConfig; 4525 - u8 aCookie[4]; 4526 - int i; 4465 + int rc; /* Return code */ 4466 + Fts5Config *pConfig = p->pConfig; /* Configuration object */ 4467 + u8 aCookie[4]; /* Binary representation of iNew */ 4527 4468 4528 4469 assert( p->rc==SQLITE_OK ); 4470 + 4529 4471 sqlite3Fts5Put32(aCookie, iNew); 4530 - for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){ 4531 - sqlite3_blob *pBlob = 0; 4532 - i64 iRowid = FTS5_STRUCTURE_ROWID(i); 4533 - rc = sqlite3_blob_open( 4534 - pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob 4535 - ); 4536 - if( rc==SQLITE_OK ){ 4537 - sqlite3_blob_write(pBlob, aCookie, 4, 0); 4538 - rc = sqlite3_blob_close(pBlob); 4539 - } 4472 + sqlite3_blob *pBlob = 0; 4473 + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, 4474 + "block", FTS5_STRUCTURE_ROWID, 1, &pBlob 4475 + ); 4476 + if( rc==SQLITE_OK ){ 4477 + sqlite3_blob_write(pBlob, aCookie, 4, 0); 4478 + rc = sqlite3_blob_close(pBlob); 4540 4479 } 4541 4480 4542 4481 return rc; 4543 4482 } 4544 4483 4545 4484 int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ 4546 4485 Fts5Structure *pStruct; 4547 - pStruct = fts5StructureRead(p, 0); 4486 + pStruct = fts5StructureRead(p); 4548 4487 fts5StructureRelease(pStruct); 4549 4488 return fts5IndexReturn(p); 4550 4489 } 4551 4490 4552 4491 4553 4492 /************************************************************************* 4554 4493 ************************************************************************** ................................................................................ 4559 4498 /* 4560 4499 ** Return a simple checksum value based on the arguments. 4561 4500 */ 4562 4501 static u64 fts5IndexEntryCksum( 4563 4502 i64 iRowid, 4564 4503 int iCol, 4565 4504 int iPos, 4566 - const char *pTerm, 4505 + int iIdx, 4506 + const char *pTerm, 4567 4507 int nTerm 4568 4508 ){ 4569 4509 int i; 4570 4510 u64 ret = iRowid; 4571 4511 ret += (ret<<3) + iCol; 4572 4512 ret += (ret<<3) + iPos; 4513 + if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); 4573 4514 for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i]; 4574 4515 return ret; 4575 4516 } 4576 4517 4577 4518 static void fts5BtreeIterInit( 4578 4519 Fts5Index *p, 4579 - int iIdx, 4580 4520 Fts5StructureSegment *pSeg, 4581 4521 Fts5BtreeIter *pIter 4582 4522 ){ 4583 4523 int nByte; 4584 4524 int i; 4585 4525 nByte = sizeof(pIter->aLvl[0]) * (pSeg->nHeight-1); 4586 4526 memset(pIter, 0, sizeof(*pIter)); 4587 4527 if( nByte ){ 4588 4528 pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte); 4589 4529 } 4590 4530 if( p->rc==SQLITE_OK ){ 4591 4531 pIter->nLvl = pSeg->nHeight-1; 4592 - pIter->iIdx = iIdx; 4593 4532 pIter->p = p; 4594 4533 pIter->pSeg = pSeg; 4595 4534 } 4596 4535 for(i=0; p->rc==SQLITE_OK && i<pIter->nLvl; i++){ 4597 - i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, i+1, 1); 4536 + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1); 4598 4537 Fts5Data *pData; 4599 4538 pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid); 4600 4539 if( pData ){ 4601 4540 fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s); 4602 4541 } 4603 4542 } 4604 4543 ................................................................................ 4631 4570 } 4632 4571 if( i==pIter->nLvl || p->rc ){ 4633 4572 pIter->bEof = 1; 4634 4573 }else{ 4635 4574 int iSegid = pIter->pSeg->iSegid; 4636 4575 for(i--; i>=0; i--){ 4637 4576 Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; 4638 - i64 iRowid = FTS5_SEGMENT_ROWID(pIter->iIdx,iSegid,i+1,pLvl[1].s.iChild); 4577 + i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild); 4639 4578 pLvl->pData = fts5DataRead(p, iRowid); 4640 4579 if( pLvl->pData ){ 4641 4580 fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s); 4642 4581 } 4643 4582 } 4644 4583 } 4645 4584 ................................................................................ 4664 4603 4665 4604 /* 4666 4605 ** This function is purely an internal test. It does not contribute to 4667 4606 ** FTS functionality, or even the integrity-check, in any way. 4668 4607 ** 4669 4608 ** Instead, it tests that the same set of pgno/rowid combinations are 4670 4609 ** visited regardless of whether the doclist-index identified by parameters 4671 -** iIdx/iSegid/iLeaf is iterated in forwards or reverse order. 4610 +** iSegid/iLeaf is iterated in forwards or reverse order. 4672 4611 */ 4673 4612 #ifdef SQLITE_DEBUG 4674 4613 static void fts5DlidxIterTestReverse( 4675 4614 Fts5Index *p, 4676 - int iIdx, /* Index to load doclist-index from */ 4677 4615 int iSegid, /* Segment id to load from */ 4678 4616 int iLeaf /* Load doclist-index for this leaf */ 4679 4617 ){ 4680 4618 Fts5DlidxIter *pDlidx = 0; 4681 4619 i64 cksum1 = 13; 4682 4620 i64 cksum2 = 13; 4683 4621 4684 - for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iLeaf); 4622 + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf); 4685 4623 fts5DlidxIterEof(p, pDlidx)==0; 4686 4624 fts5DlidxIterNext(pDlidx) 4687 4625 ){ 4688 4626 assert( pDlidx->iLeafPgno>iLeaf ); 4689 4627 cksum1 = (cksum1 ^ ( (i64)(pDlidx->iLeafPgno) << 32 )); 4690 4628 cksum1 = (cksum1 ^ pDlidx->iRowid); 4691 4629 } 4692 4630 fts5DlidxIterFree(pDlidx); 4693 4631 pDlidx = 0; 4694 4632 4695 - for(pDlidx=fts5DlidxIterInit(p, 1, iIdx, iSegid, iLeaf); 4633 + for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf); 4696 4634 fts5DlidxIterEof(p, pDlidx)==0; 4697 4635 fts5DlidxIterPrev(pDlidx) 4698 4636 ){ 4699 4637 assert( pDlidx->iLeafPgno>iLeaf ); 4700 4638 cksum2 = (cksum2 ^ ( (i64)(pDlidx->iLeafPgno) << 32 )); 4701 4639 cksum2 = (cksum2 ^ pDlidx->iRowid); 4702 4640 } 4703 4641 fts5DlidxIterFree(pDlidx); 4704 4642 pDlidx = 0; 4705 4643 4706 4644 if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; 4707 4645 } 4708 4646 #else 4709 -# define fts5DlidxIterTestReverse(w,x,y,z) 4647 +# define fts5DlidxIterTestReverse(x,y,z) 4710 4648 #endif 4711 4649 4712 4650 static void fts5IndexIntegrityCheckSegment( 4713 4651 Fts5Index *p, /* FTS5 backend object */ 4714 - int iIdx, /* Index that pSeg is a part of */ 4715 4652 Fts5StructureSegment *pSeg /* Segment to check internal consistency */ 4716 4653 ){ 4717 4654 Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ 4718 4655 4719 4656 if( pSeg->pgnoFirst==0 ) return; 4720 4657 4721 4658 /* Iterate through the b-tree hierarchy. */ 4722 - for(fts5BtreeIterInit(p, iIdx, pSeg, &iter); 4659 + for(fts5BtreeIterInit(p, pSeg, &iter); 4723 4660 p->rc==SQLITE_OK && iter.bEof==0; 4724 4661 fts5BtreeIterNext(&iter) 4725 4662 ){ 4726 4663 i64 iRow; /* Rowid for this leaf */ 4727 4664 Fts5Data *pLeaf; /* Data for this leaf */ 4728 4665 int iOff; /* Offset of first term on leaf */ 4729 4666 int i; /* Used to iterate through empty leaves */ 4730 4667 4731 4668 /* If the leaf in question has already been trimmed from the segment, 4732 4669 ** ignore this b-tree entry. Otherwise, load it into memory. */ 4733 4670 if( iter.iLeaf<pSeg->pgnoFirst ) continue; 4734 - iRow = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, iter.iLeaf); 4671 + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf); 4735 4672 pLeaf = fts5DataRead(p, iRow); 4736 4673 if( pLeaf==0 ) break; 4737 4674 4738 4675 /* Check that the leaf contains at least one term, and that it is equal 4739 4676 ** to or larger than the split-key in iter.term. */ 4740 4677 iOff = fts5GetU16(&pLeaf->p[2]); 4741 4678 if( iOff==0 ){ ................................................................................ 4767 4704 if( iter.bDlidx ){ 4768 4705 Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ 4769 4706 int iPrevLeaf = iter.iLeaf; 4770 4707 int iSegid = pSeg->iSegid; 4771 4708 int iPg; 4772 4709 i64 iKey; 4773 4710 4774 - for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iter.iLeaf); 4711 + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf); 4775 4712 fts5DlidxIterEof(p, pDlidx)==0; 4776 4713 fts5DlidxIterNext(pDlidx) 4777 4714 ){ 4778 4715 4779 4716 /* Check any rowid-less pages that occur before the current leaf. */ 4780 4717 for(iPg=iPrevLeaf+1; iPg<pDlidx->iLeafPgno; iPg++){ 4781 - iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg); 4718 + iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); 4782 4719 pLeaf = fts5DataRead(p, iKey); 4783 4720 if( pLeaf ){ 4784 4721 if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; 4785 4722 fts5DataRelease(pLeaf); 4786 4723 } 4787 4724 } 4788 4725 iPrevLeaf = pDlidx->iLeafPgno; 4789 4726 4790 4727 /* Check that the leaf page indicated by the iterator really does 4791 4728 ** contain the rowid suggested by the same. */ 4792 - iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pDlidx->iLeafPgno); 4729 + iKey = FTS5_SEGMENT_ROWID(iSegid, 0, pDlidx->iLeafPgno); 4793 4730 pLeaf = fts5DataRead(p, iKey); 4794 4731 if( pLeaf ){ 4795 4732 i64 iRowid; 4796 4733 int iRowidOff = fts5GetU16(&pLeaf->p[0]); 4797 4734 getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); 4798 4735 if( iRowid!=pDlidx->iRowid ) p->rc = FTS5_CORRUPT; 4799 4736 fts5DataRelease(pLeaf); 4800 4737 } 4801 4738 4802 4739 } 4803 4740 4804 4741 for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){ 4805 - iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg); 4742 + iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); 4806 4743 pLeaf = fts5DataRead(p, iKey); 4807 4744 if( pLeaf ){ 4808 4745 if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; 4809 4746 fts5DataRelease(pLeaf); 4810 4747 } 4811 4748 } 4812 4749 4813 4750 fts5DlidxIterFree(pDlidx); 4814 - fts5DlidxIterTestReverse(p, iIdx, iSegid, iter.iLeaf); 4751 + fts5DlidxIterTestReverse(p, iSegid, iter.iLeaf); 4815 4752 } 4816 4753 } 4817 4754 4818 4755 /* Either iter.iLeaf must be the rightmost leaf-page in the segment, or 4819 4756 ** else the segment has been completely emptied by an ongoing merge 4820 4757 ** operation. */ 4821 4758 if( p->rc==SQLITE_OK ................................................................................ 4826 4763 } 4827 4764 4828 4765 fts5BtreeIterFree(&iter); 4829 4766 } 4830 4767 4831 4768 4832 4769 static int fts5QueryCksum( 4833 - Fts5Index *p, 4834 - const char *z, 4835 - int n, 4836 - int flags, 4837 - u64 *pCksum 4770 + Fts5Index *p, /* Fts5 index object */ 4771 + int iIdx, 4772 + const char *z, /* Index key to query for */ 4773 + int n, /* Size of index key in bytes */ 4774 + int flags, /* Flags for Fts5IndexQuery */ 4775 + u64 *pCksum /* IN/OUT: Checksum value */ 4838 4776 ){ 4839 4777 u64 cksum = *pCksum; 4840 4778 Fts5IndexIter *pIdxIter = 0; 4841 4779 int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter); 4842 4780 4843 4781 while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ 4844 4782 const u8 *pPos; ................................................................................ 4849 4787 Fts5PoslistReader sReader; 4850 4788 for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader); 4851 4789 sReader.bEof==0; 4852 4790 sqlite3Fts5PoslistReaderNext(&sReader) 4853 4791 ){ 4854 4792 int iCol = FTS5_POS2COLUMN(sReader.iPos); 4855 4793 int iOff = FTS5_POS2OFFSET(sReader.iPos); 4856 - cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n); 4794 + cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); 4857 4795 } 4858 4796 rc = sqlite3Fts5IterNext(pIdxIter); 4859 4797 } 4860 4798 } 4861 4799 sqlite3Fts5IterClose(pIdxIter); 4862 4800 4863 4801 *pCksum = cksum; ................................................................................ 4871 4809 ** 4872 4810 ** Return SQLITE_CORRUPT if any of the internal checks fail, or if the 4873 4811 ** checksum does not match. Return SQLITE_OK if all checks pass without 4874 4812 ** error, or some other SQLite error code if another error (e.g. OOM) 4875 4813 ** occurs. 4876 4814 */ 4877 4815 int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ 4878 - Fts5Config *pConfig = p->pConfig; 4879 - int iIdx; /* Used to iterate through indexes */ 4880 4816 u64 cksum2 = 0; /* Checksum based on contents of indexes */ 4881 4817 u64 cksum3 = 0; /* Checksum based on contents of indexes */ 4882 4818 Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ 4883 4819 Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ 4820 + Fts5MultiSegIter *pIter; /* Used to iterate through entire index */ 4821 + Fts5Structure *pStruct; /* Index structure */ 4822 + 4823 + /* Load the FTS index structure */ 4824 + pStruct = fts5StructureRead(p); 4884 4825 4885 4826 /* Check that the internal nodes of each segment match the leaves */ 4886 - for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){ 4887 - Fts5Structure *pStruct = fts5StructureRead(p, iIdx); 4888 - if( pStruct ){ 4889 - int iLvl, iSeg; 4890 - for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 4891 - for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 4892 - Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; 4893 - fts5IndexIntegrityCheckSegment(p, iIdx, pSeg); 4894 - } 4827 + if( pStruct ){ 4828 + int iLvl, iSeg; 4829 + for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){ 4830 + for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){ 4831 + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; 4832 + fts5IndexIntegrityCheckSegment(p, pSeg); 4895 4833 } 4896 4834 } 4897 - fts5StructureRelease(pStruct); 4898 4835 } 4899 4836 4900 4837 /* The cksum argument passed to this function is a checksum calculated 4901 4838 ** based on all expected entries in the FTS index (including prefix index 4902 4839 ** entries). This block checks that a checksum calculated based on the 4903 4840 ** actual contents of FTS index is identical. 4904 4841 ** ................................................................................ 4906 4843 ** variable cksum2) based on entries extracted from the full-text index 4907 4844 ** while doing a linear scan of each individual index in turn. 4908 4845 ** 4909 4846 ** As each term visited by the linear scans, a separate query for the 4910 4847 ** same term is performed. cksum3 is calculated based on the entries 4911 4848 ** extracted by these queries. 4912 4849 */ 4913 - for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){ 4914 - Fts5MultiSegIter *pIter; 4915 - Fts5Structure *pStruct = fts5StructureRead(p, iIdx); 4916 - for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter); 4917 - fts5MultiIterEof(p, pIter)==0; 4918 - fts5MultiIterNext(p, pIter, 0, 0) 4919 - ){ 4920 - int n; /* Size of term in bytes */ 4921 - i64 iPos = 0; /* Position read from poslist */ 4922 - int iOff = 0; /* Offset within poslist */ 4923 - i64 iRowid = fts5MultiIterRowid(pIter); 4924 - char *z = (char*)fts5MultiIterTerm(pIter, &n); 4925 - 4926 - poslist.n = 0; 4927 - fts5MultiIterPoslist(p, pIter, 0, &poslist); 4928 - while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ 4929 - int iCol = FTS5_POS2COLUMN(iPos); 4930 - int iTokOff = FTS5_POS2OFFSET(iPos); 4931 - cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n); 4850 + for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter); 4851 + fts5MultiIterEof(p, pIter)==0; 4852 + fts5MultiIterNext(p, pIter, 0, 0) 4853 + ){ 4854 + int n; /* Size of term in bytes */ 4855 + i64 iPos = 0; /* Position read from poslist */ 4856 + int iOff = 0; /* Offset within poslist */ 4857 + i64 iRowid = fts5MultiIterRowid(pIter); 4858 + char *z = (char*)fts5MultiIterTerm(pIter, &n); 4859 + 4860 + poslist.n = 0; 4861 + fts5MultiIterPoslist(p, pIter, 0, &poslist); 4862 + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ 4863 + int iCol = FTS5_POS2COLUMN(iPos); 4864 + int iTokOff = FTS5_POS2OFFSET(iPos); 4865 + cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); 4866 + } 4867 + 4868 + /* If this is a new term, query for it. Update cksum3 with the results. */ 4869 + if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){ 4870 + const char *zTerm = &z[1]; /* The term without the prefix-byte */ 4871 + int nTerm = n-1; /* Size of zTerm in bytes */ 4872 + int iIdx = (z[0] - FTS5_MAIN_PREFIX); 4873 + int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); 4874 + int rc; 4875 + u64 ck1 = 0; 4876 + u64 ck2 = 0; 4877 + 4878 + /* Check that the results returned for ASC and DESC queries are 4879 + ** the same. If not, call this corruption. */ 4880 + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); 4881 + if( rc==SQLITE_OK ){ 4882 + int f = flags|FTS5INDEX_QUERY_DESC; 4883 + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); 4884 + } 4885 + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; 4886 + 4887 + /* If this is a prefix query, check that the results returned if the 4888 + ** the index is disabled are the same. In both ASC and DESC order. */ 4889 + if( iIdx>0 && rc==SQLITE_OK ){ 4890 + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; 4891 + ck2 = 0; 4892 + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); 4893 + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; 4894 + } 4895 + if( iIdx>0 && rc==SQLITE_OK ){ 4896 + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; 4897 + ck2 = 0; 4898 + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); 4899 + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; 4932 4900 } 4933 4901 4934 - /* If this is a new term, query for it. Update cksum3 with the results. */ 4935 - if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){ 4936 - int rc; 4937 - int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); 4938 - u64 ck1 = 0; 4939 - u64 ck2 = 0; 4940 - 4941 - /* Check that the results returned for ASC and DESC queries are 4942 - ** the same. If not, call this corruption. */ 4943 - rc = fts5QueryCksum(p, z, n, flags, &ck1); 4944 - if( rc==SQLITE_OK ){ 4945 - rc = fts5QueryCksum(p, z, n, flags|FTS5INDEX_QUERY_DESC, &ck2); 4946 - } 4947 - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; 4948 - 4949 - /* If this is a prefix query, check that the results returned if the 4950 - ** the index is disabled are the same. In both ASC and DESC order. */ 4951 - if( iIdx>0 && rc==SQLITE_OK ){ 4952 - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; 4953 - ck2 = 0; 4954 - rc = fts5QueryCksum(p, z, n, f, &ck2); 4955 - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; 4956 - } 4957 - if( iIdx>0 && rc==SQLITE_OK ){ 4958 - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; 4959 - ck2 = 0; 4960 - rc = fts5QueryCksum(p, z, n, f, &ck2); 4961 - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; 4962 - } 4963 - 4964 - cksum3 ^= ck1; 4965 - fts5BufferSet(&rc, &term, n, (const u8*)z); 4966 - p->rc = rc; 4967 - } 4902 + cksum3 ^= ck1; 4903 + fts5BufferSet(&rc, &term, n, (const u8*)z); 4904 + p->rc = rc; 4968 4905 } 4969 - fts5MultiIterFree(p, pIter); 4970 - fts5StructureRelease(pStruct); 4971 4906 } 4907 + fts5MultiIterFree(p, pIter); 4908 + 4972 4909 if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; 4973 4910 if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT; 4974 4911 4912 + fts5StructureRelease(pStruct); 4975 4913 fts5BufferFree(&term); 4976 4914 fts5BufferFree(&poslist); 4977 4915 return fts5IndexReturn(p); 4978 4916 } 4979 4917 4980 4918 4981 4919 /* ................................................................................ 4989 4927 int iCol, /* Column term appears in */ 4990 4928 int iPos, /* Position term appears in */ 4991 4929 const char *pTerm, int nTerm /* Term at iPos */ 4992 4930 ){ 4993 4931 u64 ret = 0; /* Return value */ 4994 4932 int iIdx; /* For iterating through indexes */ 4995 4933 4996 - ret = fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nTerm); 4934 + ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm); 4997 4935 4998 4936 for(iIdx=0; iIdx<pConfig->nPrefix; iIdx++){ 4999 4937 int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]); 5000 4938 if( nByte ){ 5001 - ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nByte); 4939 + ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte); 5002 4940 } 5003 4941 } 5004 4942 5005 4943 return ret; 5006 4944 } 5007 4945 5008 4946 /************************************************************************* ................................................................................ 5013 4951 5014 4952 /* 5015 4953 ** Decode a segment-data rowid from the %_data table. This function is 5016 4954 ** the opposite of macro FTS5_SEGMENT_ROWID(). 5017 4955 */ 5018 4956 static void fts5DecodeRowid( 5019 4957 i64 iRowid, /* Rowid from %_data table */ 5020 - int *piIdx, /* OUT: Index */ 5021 4958 int *piSegid, /* OUT: Segment id */ 5022 4959 int *piHeight, /* OUT: Height */ 5023 4960 int *piPgno /* OUT: Page number */ 5024 4961 ){ 5025 4962 *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1)); 5026 4963 iRowid >>= FTS5_DATA_PAGE_B; 5027 4964 5028 4965 *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1)); 5029 4966 iRowid >>= FTS5_DATA_HEIGHT_B; 5030 4967 5031 4968 *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); 5032 - iRowid >>= FTS5_DATA_ID_B; 5033 - 5034 - *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1)); 5035 4969 } 5036 4970 5037 4971 static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ 5038 - int iIdx,iSegid,iHeight,iPgno; /* Rowid compenents */ 5039 - fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno); 4972 + int iSegid, iHeight, iPgno; /* Rowid compenents */ 4973 + fts5DecodeRowid(iKey, &iSegid, &iHeight, &iPgno); 5040 4974 5041 4975 if( iSegid==0 ){ 5042 4976 if( iKey==FTS5_AVERAGES_ROWID ){ 5043 4977 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) "); 5044 4978 }else{ 5045 4979 sqlite3Fts5BufferAppendPrintf(pRc, pBuf, 5046 4980 "{structure idx=%d}", (int)(iKey-10) 5047 4981 ); 5048 4982 } 5049 4983 } 5050 4984 else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){ 5051 - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)", 5052 - iIdx, iSegid, iPgno 4985 + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx segid=%d pgno=%d)", 4986 + iSegid, iPgno 5053 4987 ); 5054 4988 }else{ 5055 - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)", 5056 - iIdx, iSegid, iHeight, iPgno 4989 + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(segid=%d h=%d pgno=%d)", 4990 + iSegid, iHeight, iPgno 5057 4991 ); 5058 4992 } 5059 4993 } 5060 4994 5061 4995 static void fts5DebugStructure( 5062 4996 int *pRc, /* IN/OUT: error code */ 5063 4997 Fts5Buffer *pBuf, ................................................................................ 5159 5093 */ 5160 5094 static void fts5DecodeFunction( 5161 5095 sqlite3_context *pCtx, /* Function call context */ 5162 5096 int nArg, /* Number of args (always 2) */ 5163 5097 sqlite3_value **apVal /* Function arguments */ 5164 5098 ){ 5165 5099 i64 iRowid; /* Rowid for record being decoded */ 5166 - int iIdx,iSegid,iHeight,iPgno; /* Rowid components */ 5100 + int iSegid,iHeight,iPgno; /* Rowid components */ 5167 5101 const u8 *aBlob; int n; /* Record to decode */ 5168 5102 u8 *a = 0; 5169 5103 Fts5Buffer s; /* Build up text to return here */ 5170 5104 int rc = SQLITE_OK; /* Return code */ 5171 5105 int nSpace = 0; 5172 5106 5173 5107 assert( nArg==2 ); ................................................................................ 5176 5110 n = sqlite3_value_bytes(apVal[1]); 5177 5111 aBlob = sqlite3_value_blob(apVal[1]); 5178 5112 5179 5113 nSpace = n + FTS5_DATA_ZERO_PADDING; 5180 5114 a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); 5181 5115 if( a==0 ) goto decode_out; 5182 5116 memcpy(a, aBlob, n); 5183 - fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno); 5117 + fts5DecodeRowid(iRowid, &iSegid, &iHeight, &iPgno); 5184 5118 5185 5119 fts5DebugRowid(&rc, &s, iRowid); 5186 5120 if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){ 5187 5121 Fts5Data dlidx; 5188 5122 Fts5DlidxIter iter; 5189 5123 5190 5124 dlidx.p = a; ................................................................................ 5297 5231 const char *zArg; 5298 5232 if( nArg==0 ){ 5299 5233 sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1); 5300 5234 }else{ 5301 5235 zArg = (const char*)sqlite3_value_text(apVal[0]); 5302 5236 if( 0==sqlite3_stricmp(zArg, "segment") ){ 5303 5237 i64 iRowid; 5304 - int idx, segid, height, pgno; 5305 - if( nArg!=5 ){ 5238 + int segid, height, pgno; 5239 + if( nArg!=4 ){ 5306 5240 sqlite3_result_error(pCtx, 5307 - "should be: fts5_rowid('segment', idx, segid, height, pgno))", -1 5241 + "should be: fts5_rowid('segment', segid, height, pgno))", -1 5308 5242 ); 5309 5243 }else{ 5310 - idx = sqlite3_value_int(apVal[1]); 5311 - segid = sqlite3_value_int(apVal[2]); 5312 - height = sqlite3_value_int(apVal[3]); 5313 - pgno = sqlite3_value_int(apVal[4]); 5314 - iRowid = FTS5_SEGMENT_ROWID(idx, segid, height, pgno); 5244 + segid = sqlite3_value_int(apVal[1]); 5245 + height = sqlite3_value_int(apVal[2]); 5246 + pgno = sqlite3_value_int(apVal[3]); 5247 + iRowid = FTS5_SEGMENT_ROWID(segid, height, pgno); 5315 5248 sqlite3_result_int64(pCtx, iRowid); 5316 5249 } 5250 +#if 0 5317 5251 }else if( 0==sqlite3_stricmp(zArg, "start-of-index") ){ 5318 5252 i64 iRowid; 5319 5253 int idx; 5320 5254 if( nArg!=2 ){ 5321 5255 sqlite3_result_error(pCtx, 5322 5256 "should be: fts5_rowid('start-of-index', idx)", -1 5323 5257 ); 5324 5258 }else{ 5325 5259 idx = sqlite3_value_int(apVal[1]); 5326 5260 iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0); 5327 5261 sqlite3_result_int64(pCtx, iRowid); 5328 5262 } 5263 +#endif 5329 5264 }else { 5330 5265 sqlite3_result_error(pCtx, 5331 5266 "first arg to fts5_rowid() must be 'segment' " 5332 5267 "or 'start-of-index'" 5333 5268 , -1 5334 5269 ); 5335 5270 }
Changes to ext/fts5/fts5_storage.c.
253 253 ); 254 254 } 255 255 if( rc==SQLITE_OK ){ 256 256 rc = sqlite3Fts5CreateTable( 257 257 pConfig, "config", "k PRIMARY KEY, v", 1, pzErr 258 258 ); 259 259 } 260 + if( rc==SQLITE_OK ){ 261 + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); 262 + } 260 263 } 261 264 262 265 if( rc ){ 263 266 sqlite3Fts5StorageClose(p); 264 267 *pp = 0; 265 268 } 266 269 return rc; ................................................................................ 539 542 ); 540 543 541 544 /* Reinitialize the %_data table. This call creates the initial structure 542 545 ** and averages records. */ 543 546 if( rc==SQLITE_OK ){ 544 547 rc = sqlite3Fts5IndexReinit(p->pIndex); 545 548 } 549 + if( rc==SQLITE_OK ){ 550 + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); 551 + } 546 552 return rc; 547 553 } 548 554 549 555 int sqlite3Fts5StorageRebuild(Fts5Storage *p){ 550 556 Fts5Buffer buf = {0,0,0}; 551 557 Fts5Config *pConfig = p->pConfig; 552 558 sqlite3_stmt *pScan = 0; ................................................................................ 979 985 int sqlite3Fts5StorageRollback(Fts5Storage *p){ 980 986 p->bTotalsValid = 0; 981 987 return sqlite3Fts5IndexRollback(p->pIndex); 982 988 } 983 989 984 990 int sqlite3Fts5StorageConfigValue( 985 991 Fts5Storage *p, 986 - const char *z, 987 - sqlite3_value *pVal 992 + const char *z, 993 + sqlite3_value *pVal, 994 + int iVal 988 995 ){ 989 996 sqlite3_stmt *pReplace = 0; 990 997 int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); 991 998 if( rc==SQLITE_OK ){ 992 999 sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC); 993 - sqlite3_bind_value(pReplace, 2, pVal); 1000 + if( pVal ){ 1001 + sqlite3_bind_value(pReplace, 2, pVal); 1002 + }else{ 1003 + sqlite3_bind_int(pReplace, 2, iVal); 1004 + } 994 1005 sqlite3_step(pReplace); 995 1006 rc = sqlite3_reset(pReplace); 996 1007 } 997 - if( rc==SQLITE_OK ){ 1008 + if( rc==SQLITE_OK && pVal ){ 998 1009 int iNew = p->pConfig->iCookie + 1; 999 1010 rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); 1000 1011 if( rc==SQLITE_OK ){ 1001 1012 p->pConfig->iCookie = iNew; 1002 1013 } 1003 1014 } 1004 1015 return rc; 1005 1016 } 1006 1017 1007 1018 1008 1019 #endif /* SQLITE_ENABLE_FTS5 */
Changes to ext/fts5/test/fts5al.test.
22 22 finish_test 23 23 return 24 24 } 25 25 26 26 do_execsql_test 1.1 { 27 27 CREATE VIRTUAL TABLE ft1 USING fts5(x); 28 28 SELECT * FROM ft1_config; 29 -} {} 29 +} {version 1} 30 30 31 31 do_execsql_test 1.2 { 32 32 INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); 33 33 SELECT * FROM ft1_config; 34 -} {pgsz 32} 34 +} {pgsz 32 version 1} 35 35 36 36 do_execsql_test 1.3 { 37 37 INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); 38 38 SELECT * FROM ft1_config; 39 -} {pgsz 64} 39 +} {pgsz 64 version 1} 40 40 41 41 #-------------------------------------------------------------------------- 42 42 # Test the logic for parsing the rank() function definition. 43 43 # 44 44 foreach {tn defn} { 45 45 1 "fname()" 46 46 2 "fname(1)"
Changes to ext/fts5/test/fts5corrupt.test.
33 33 db_save 34 34 35 35 do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') } 36 36 set segid [lindex [fts5_level_segids t1] 0] 37 37 38 38 do_test 1.3 { 39 39 execsql { 40 - DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4); 40 + DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 0, 4); 41 41 } 42 42 catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } 43 43 } {1 {database disk image is malformed}} 44 44 45 45 do_test 1.4 { 46 46 db_restore_and_reopen 47 47 execsql { 48 48 UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE 49 - rowid = fts5_rowid('segment', 0, $segid, 0, 4); 49 + rowid = fts5_rowid('segment', $segid, 0, 4); 50 50 } 51 51 catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } 52 52 } {1 {database disk image is malformed}} 53 53 54 54 db_restore_and_reopen 55 55 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r} 56 56
Changes to ext/fts5/test/fts5corrupt2.test.
122 122 INSERT INTO x3(x3, rank) VALUES('pgsz', 32); 123 123 WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000) 124 124 INSERT INTO x3 125 125 SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii; 126 126 } 127 127 128 128 foreach {tn hdr} { 129 - 1 "\00\00\00\00" 130 - 2 "\FF\FF\FF\FF" 129 + 1 "\x00\x00\x00\x00" 130 + 2 "\xFF\xFF\xFF\xFF" 131 131 } { 132 132 set tn2 0 133 133 set nCorrupt 0 134 134 foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] { 135 135 if {$rowid & $mask} continue 136 136 incr tn2 137 137 do_test 3.$tn.$tn2 {
Changes to ext/fts5/test/fts5ea.test.
74 74 11 {a AND "abc} {unterminated string} 75 75 76 76 12 {NEAR(a b, xyz)} {expected integer, got "xyz"} 77 77 13 {NEAR(a b, // )} {expected integer, got "//"} 78 78 } { 79 79 do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err] 80 80 } 81 + 82 +#------------------------------------------------------------------------- 83 +# Experiment with a tokenizer that considers " to be a token character. 84 +# 85 +do_execsql_test 4.0 { 86 + SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"'); 87 +} {{"a" AND """"}} 81 88 82 89 83 90 84 91 85 92 finish_test
Changes to ext/fts5/test/fts5fault1.test.
28 28 # 2: INSERT statement 29 29 # 3: DELETE statement 30 30 # 4: MATCH expressions 31 31 # 32 32 # 33 33 34 34 faultsim_save_and_close 35 -do_faultsim_test 1 -prep { 35 +do_faultsim_test 1 -faults ioerr-t* -prep { 36 36 faultsim_restore_and_reopen 37 37 } -body { 38 38 execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') } 39 39 } -test { 40 - faultsim_test_result {0 {}} 40 + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} 41 41 } 42 42 43 43 reset_db 44 44 do_execsql_test 2.0 { 45 45 CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3'); 46 46 } 47 47 faultsim_save_and_close
Changes to ext/fts5/test/fts5fault4.test.
197 197 198 198 do_faultsim_test 6.2 -faults oom-t* -body { 199 199 db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' } 200 200 } -test { 201 201 faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM} 202 202 } 203 203 204 -} 205 - 206 204 #------------------------------------------------------------------------- 207 205 # OOM error when querying for a phrase with many tokens. 208 206 # 209 207 reset_db 210 208 do_execsql_test 7.0 { 211 209 CREATE VIRTUAL TABLE tt USING fts5(x, y); 212 210 INSERT INTO tt VALUES('f b g b c b', 'f a d c c b'); -- 1 ................................................................................ 299 297 } 300 298 301 299 do_faultsim_test 9.1 -faults oom-* -body { 302 300 db eval { SELECT rowid FROM tt WHERE tt MATCH 'a NOT x' } 303 301 } -test { 304 302 faultsim_test_result {0 {50 100 150 200}} {1 SQLITE_NOMEM} 305 303 } 304 + 305 +#------------------------------------------------------------------------- 306 +# OOM in fts5_expr() SQL function. 307 +# 308 +reset_db 309 +do_execsql_test 10.0 { 310 + CREATE VIRTUAL TABLE tt USING fts5(x); 311 + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); 312 + BEGIN; 313 + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<200) 314 + INSERT INTO tt(rowid, x) 315 + SELECT i, CASE WHEN (i%50)==0 THEN 'a a a a a a' ELSE 'a x a x a x' END 316 + FROM ii; 317 + COMMIT; 318 +} 319 + 320 +} 321 + 322 +do_faultsim_test 10.1 -faults oom-t* -body { 323 + db one { SELECT fts5_expr('a AND b NEAR(a b)') } 324 +} -test { 325 + faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}} 326 +} 327 + 328 +#do_faultsim_test 10.2 -faults oom-t* -body { 329 +# db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') } 330 +#} -test { 331 +# set res {[ns -col 0 -- {a b c}] && ([ns -- {b}] && [ns -near 10 -- {a} {b}]} 332 +# faultsim_test_result [list 0 $res] 333 +#} 306 334 307 335 308 336 309 337 finish_test 310 338
Added ext/fts5/test/fts5integrity.test.
1 +# 2015 Jan 13 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#*********************************************************************** 11 +# 12 +# This file containst tests focused on the integrity-check procedure. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5integrity 17 + 18 +do_execsql_test 1.0 { 19 + CREATE VIRTUAL TABLE xx USING fts5(x); 20 + INSERT INTO xx VALUES('term'); 21 +} 22 +do_execsql_test 1.1 { 23 + INSERT INTO xx(xx) VALUES('integrity-check'); 24 +} 25 + 26 +do_execsql_test 2.0 { 27 + CREATE VIRTUAL TABLE yy USING fts5(x, prefix=1); 28 + INSERT INTO yy VALUES('term'); 29 +} 30 +do_execsql_test 2.1 { 31 + INSERT INTO yy(yy) VALUES('integrity-check'); 32 +} 33 + 34 +finish_test 35 +
Changes to ext/fts5/test/fts5prefix.test.
11 11 # 12 12 # This file containst tests focused on prefix indexes. 13 13 # 14 14 15 15 source [file join [file dirname [info script]] fts5_common.tcl] 16 16 set testprefix fts5prefix 17 17 18 +do_execsql_test 1.0 { 19 + CREATE VIRTUAL TABLE xx USING fts5(x, prefix=1); 20 + INSERT INTO xx VALUES('one two three'); 21 + INSERT INTO xx VALUES('four five six'); 22 + INSERT INTO xx VALUES('seven eight nine ten'); 23 +} 24 + 25 +do_execsql_test 1.1 { 26 + SELECT rowid FROM xx WHERE xx MATCH 't*' 27 +} {1 3} 28 + 18 29 19 30 #------------------------------------------------------------------------- 20 31 # Check that prefix indexes really do index n-character prefixes, not 21 32 # n-byte prefixes. Use the ascii tokenizer so as not to be confused by 22 33 # diacritic removal. 23 34 # 24 -do_execsql_test 1.0 { 35 +do_execsql_test 2.0 { 25 36 CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = ascii, prefix = 2) 26 37 } 27 38 28 -do_test 1.2 { 39 +do_test 2.1 { 29 40 foreach {rowid string} { 30 41 1 "\xCA\xCB\xCC\xCD" 31 42 2 "\u1234\u5678\u4321\u8765" 32 43 } { 33 44 execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $string) } 34 45 } 35 46 } {} 36 47 37 -do_execsql_test 1.1.2 { 48 +do_execsql_test 2.2 { 38 49 INSERT INTO t1(t1) VALUES('integrity-check'); 39 50 } 40 51 41 -#db eval { select fts5_decode(id, block) AS d FROM t1_data; } { puts $d } 42 - 43 -foreach o {1 2} { 44 - if {$o==2} breakpoint 45 - foreach {tn q res} { 46 - 1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1 47 - 2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2 48 - } { 49 - do_execsql_test 1.$o.$tn $q $res 50 - } 51 - 52 - execsql { 53 - DELETE FROM t1_data WHERE 54 - rowid>=fts5_rowid('start-of-index', 0) AND 55 - rowid<fts5_rowid('start-of-index', 1); 56 - } 52 +foreach {tn q res} { 53 + 1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1 54 + 2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2 55 +} { 56 + do_execsql_test 2.3.$tn $q $res 57 57 } 58 58 59 59 60 60 finish_test 61 61
Changes to ext/fts5/test/fts5rowid.test.
17 17 18 18 do_catchsql_test 1.1 { 19 19 SELECT fts5_rowid() 20 20 } {1 {should be: fts5_rowid(subject, ....)}} 21 21 22 22 do_catchsql_test 1.2 { 23 23 SELECT fts5_rowid('segment') 24 -} {1 {should be: fts5_rowid('segment', idx, segid, height, pgno))}} 24 +} {1 {should be: fts5_rowid('segment', segid, height, pgno))}} 25 25 26 26 do_execsql_test 1.3 { 27 - SELECT fts5_rowid('segment', 1, 1, 1, 1) 28 -} {4503670494330881} 29 - 30 -do_catchsql_test 1.4 { 31 - SELECT fts5_rowid('start-of-index'); 32 -} {1 {should be: fts5_rowid('start-of-index', idx)}} 33 - 34 -do_execsql_test 1.5 { 35 - SELECT fts5_rowid('start-of-index', 1); 36 -} {4503668346847232} 27 + SELECT fts5_rowid('segment', 1, 1, 1) 28 +} {70866960385} 37 29 38 30 do_catchsql_test 1.4 { 39 31 SELECT fts5_rowid('nosucharg'); 40 32 } {1 {first arg to fts5_rowid() must be 'segment' or 'start-of-index'}} 41 33 42 34 43 35 #-------------------------------------------------------------------------
Changes to ext/fts5/tool/loadfts5.tcl.
36 36 puts stderr " -fts4 (use fts4 instead of fts5)" 37 37 puts stderr " -fts5 (use fts5)" 38 38 puts stderr " -porter (use porter tokenizer)" 39 39 puts stderr " -delete (delete the database file before starting)" 40 40 puts stderr " -limit N (load no more than N documents)" 41 41 puts stderr " -automerge N (set the automerge parameter to N)" 42 42 puts stderr " -crisismerge N (set the crisismerge parameter to N)" 43 + puts stderr " -prefix PREFIX (comma separated prefix= argument)" 43 44 exit 1 44 45 } 45 46 46 47 set O(vtab) fts5 47 48 set O(tok) "" 48 49 set O(limit) 0 49 50 set O(delete) 0 50 51 set O(automerge) -1 51 52 set O(crisismerge) -1 53 +set O(prefix) "" 52 54 53 55 if {[llength $argv]<2} usage 54 56 set nOpt [expr {[llength $argv]-2}] 55 57 for {set i 0} {$i < $nOpt} {incr i} { 56 58 set arg [lindex $argv $i] 57 59 switch -- [lindex $argv $i] { 58 60 -fts4 { ................................................................................ 81 83 set O(automerge) [lindex $argv $i] 82 84 } 83 85 84 86 -crisismerge { 85 87 if { [incr i]>=$nOpt } usage 86 88 set O(crisismerge) [lindex $argv $i] 87 89 } 90 + 91 + -prefix { 92 + if { [incr i]>=$nOpt } usage 93 + set O(prefix) [lindex $argv $i] 94 + } 88 95 89 96 default { 90 97 usage 91 98 } 92 99 } 93 100 } 94 101 95 102 set dbfile [lindex $argv end-1] 96 103 if {$O(delete)} { file delete -force $dbfile } 97 104 sqlite3 db $dbfile 98 105 db func loadfile loadfile 99 106 100 107 db transaction { 108 + set pref "" 109 + if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } 101 110 catch { 102 - db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" 111 + db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)" 103 112 } 104 113 if {$O(automerge)>=0} { 105 114 if {$O(vtab) == "fts5"} { 106 115 db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } 107 116 } else { 108 117 db eval { INSERT INTO t1(t1) VALUES('automerge=' || $O(automerge)) } 109 118 }