Index: ext/fts5/fts5.c ================================================================== --- ext/fts5/fts5.c +++ ext/fts5/fts5.c @@ -1162,11 +1162,11 @@ } if( rc==SQLITE_OK ){ if( bError ){ rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal); + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal, 0); } } } return rc; } Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -119,14 +119,18 @@ int nAutomerge; /* 'automerge' setting */ int nCrisisMerge; /* Maximum allowed segments per level */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ }; + +/* Current expected value of %_config table 'version' field */ +#define FTS5_CURRENT_VERSION 1 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 + int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** @@ -392,10 +396,11 @@ int sqlite3Fts5HashWrite( Fts5Hash*, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ + char bByte, const char *pToken, int nToken /* Token to add or remove to or from index */ ); /* ** Empty (but do not delete) a hash table. @@ -456,11 +461,13 @@ int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); int sqlite3Fts5StorageRollback(Fts5Storage *p); -int sqlite3Fts5StorageConfigValue(Fts5Storage *p, const char*, sqlite3_value*); +int sqlite3Fts5StorageConfigValue( + Fts5Storage *p, const char*, sqlite3_value*, int +); int sqlite3Fts5StorageSpecialDelete(Fts5Storage *p, i64 iDel, sqlite3_value**); int sqlite3Fts5StorageDeleteAll(Fts5Storage *p); int sqlite3Fts5StorageRebuild(Fts5Storage *p); Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -201,37 +201,10 @@ if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ fts5Dequote(z); } } -/* -** Argument z points to a nul-terminated string containing an SQL identifier. -** This function returns a copy of the identifier enclosed in backtick -** quotes. -*/ -static char *fts5EscapeName(int *pRc, const char *z){ - char *pRet = 0; - if( *pRc==SQLITE_OK ){ - int n = strlen(z); - pRet = (char*)sqlite3_malloc(2 + 2*n + 1); - if( pRet==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - int i; - char *p = pRet; - *p++ = '`'; - for(i=0; izContentRowid); if( p->eContent!=FTS5_CONTENT_NONE ){ for(i=0; inCol; i++){ if( p->eContent==FTS5_CONTENT_EXTERNAL ){ @@ -847,10 +819,11 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ const char *zSelect = "SELECT k, v FROM %Q.'%q_config'"; char *zSql; sqlite3_stmt *p = 0; int rc; + int iVersion = 0; /* Set default values */ pConfig->pgsz = FTS5_DEFAULT_PAGE_SIZE; pConfig->nAutomerge = FTS5_DEFAULT_AUTOMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; @@ -866,17 +839,25 @@ assert( rc==SQLITE_OK || p==0 ); if( rc==SQLITE_OK ){ while( SQLITE_ROW==sqlite3_step(p) ){ const char *zK = (const char*)sqlite3_column_text(p, 0); sqlite3_value *pVal = sqlite3_column_value(p, 1); - sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0); + if( 0==sqlite3_stricmp(zK, "version") ){ + iVersion = sqlite3_value_int(pVal); + }else{ + sqlite3Fts5ConfigSetValue(pConfig, zK, pVal, 0); + } } - rc = sqlite3_finalize(p); + if( rc==SQLITE_OK ) rc = sqlite3_finalize(p); + } + + if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + rc = sqlite3Fts5Corrupt(); } if( rc==SQLITE_OK ){ pConfig->iCookie = iCookie; } return rc; } #endif /* SQLITE_ENABLE_FTS5 */ Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -1369,11 +1369,11 @@ char *zNew; va_list ap; va_start(ap, zFmt); zNew = sqlite3_vmprintf(zFmt, ap); va_end(ap); - if( zApp ){ + if( zApp && zNew ){ char *zNew2 = sqlite3_mprintf("%s%s", zApp, zNew); sqlite3_free(zNew); zNew = zNew2; } sqlite3_free(zApp); @@ -1546,27 +1546,30 @@ const char **azConfig; /* Array of arguments for Fts5Config */ const char *zNearsetCmd = "nearset"; int nConfig; /* Size of azConfig[] */ Fts5Config *pConfig = 0; + int iArg = 1; if( bTcl && nArg>1 ){ zNearsetCmd = (const char*)sqlite3_value_text(apVal[1]); + iArg = 2; } - nConfig = nArg + 2 - bTcl; + nConfig = 3 + (nArg-iArg); azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig); if( azConfig==0 ){ sqlite3_result_error_nomem(pCtx); return; } azConfig[0] = 0; azConfig[1] = "main"; azConfig[2] = "tbl"; - for(i=1+bTcl; ipRoot); }else{ zText = fts5ExprPrint(pConfig, pExpr->pRoot); } - if( rc==SQLITE_OK ){ + if( zText==0 ){ + rc = SQLITE_NOMEM; + }else{ sqlite3_result_text(pCtx, zText, -1, SQLITE_TRANSIENT); sqlite3_free(zText); } } Index: ext/fts5/fts5_hash.c ================================================================== --- ext/fts5/fts5_hash.c +++ ext/fts5/fts5_hash.c @@ -133,10 +133,20 @@ for(i=n-1; i>=0; i--){ h = (h << 3) ^ h ^ p[i]; } return (h % nSlot); } + +static unsigned int fts5HashKey2(int nSlot, char b, const char *p, int n){ + int i; + unsigned int h = 13; + for(i=n-1; i>=0; i--){ + h = (h << 3) ^ h ^ p[i]; + } + h = (h << 3) ^ h ^ b; + return (h % nSlot); +} /* ** Resize the hash table by doubling the number of slots. */ static int fts5HashResize(Fts5Hash *pHash){ @@ -189,40 +199,48 @@ int sqlite3Fts5HashWrite( Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ + char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ - unsigned int iHash = fts5HashKey(pHash->nSlot, pToken, nToken); + unsigned int iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ /* Attempt to locate an existing hash entry */ for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ - if( memcmp(p->zKey, pToken, nToken)==0 && p->zKey[nToken]==0 ) break; + if( p->zKey[0]==bByte + && memcmp(&p->zKey[1], pToken, nToken)==0 + && p->zKey[nToken+1]==0 + ){ + break; + } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ - int nByte = sizeof(Fts5HashEntry) + nToken + 1 + 64; + int nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; - iHash = fts5HashKey(pHash->nSlot, pToken, nToken); + iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); } p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, sizeof(Fts5HashEntry)); p->nAlloc = nByte; - memcpy(p->zKey, pToken, nToken); - p->zKey[nToken] = '\0'; - p->nData = nToken + 1 + sizeof(Fts5HashEntry); + p->zKey[0] = bByte; + memcpy(&p->zKey[1], pToken, nToken); + assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) ); + p->zKey[nToken+1] = '\0'; + p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry); p->nData += sqlite3PutVarint(&((u8*)p)[p->nData], iRowid); p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; p->pHashNext = pHash->aSlot[iHash]; Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -47,10 +47,16 @@ #define FTS5_OPT_WORK_UNIT 1000 /* Number of leaf pages per optimize step */ #define FTS5_WORK_UNIT 64 /* Number of leaf pages in unit of work */ #define FTS5_MIN_DLIDX_SIZE 4 /* Add dlidx if this many empty pages */ +#define FTS5_MAIN_PREFIX '0' + +#if FTS5_MAX_PREFIX_INDEXES > 31 +# error "FTS5_MAX_PREFIX_INDEXES is too large" +#endif + /* ** Details: ** ** The %_data table managed by this module, ** @@ -209,11 +215,11 @@ /* ** Rowids for the averages and structure records in the %_data table. */ #define FTS5_AVERAGES_ROWID 1 /* Rowid used for the averages record */ -#define FTS5_STRUCTURE_ROWID(iIdx) (10 + (iIdx)) /* For structure records */ +#define FTS5_STRUCTURE_ROWID 10 /* The structure record */ /* ** Macros determining the rowids used by segment nodes. All nodes in all ** segments for all indexes (the regular FTS index and any prefix indexes) ** are stored in the %_data table with large positive rowids. @@ -231,26 +237,20 @@ ** The rowid for a node is then found using the FTS5_SEGMENT_ROWID() macro ** below. The FTS5_SEGMENT_*_BITS macros define the number of bits used ** to encode the three FTS5_SEGMENT_ROWID() arguments. This module returns ** SQLITE_FULL and fails the current operation if they ever prove too small. */ -#define FTS5_DATA_IDX_B 5 /* Max of 31 prefix indexes */ #define FTS5_DATA_ID_B 16 /* Max seg id number 65535 */ #define FTS5_DATA_HEIGHT_B 5 /* Max b-tree height of 32 */ #define FTS5_DATA_PAGE_B 31 /* Max page number of 2147483648 */ -#define FTS5_SEGMENT_ROWID(idx, segid, height, pgno) ( \ - ((i64)(idx) << (FTS5_DATA_ID_B + FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ +#define FTS5_SEGMENT_ROWID(segid, height, pgno) ( \ ((i64)(segid) << (FTS5_DATA_PAGE_B + FTS5_DATA_HEIGHT_B)) + \ ((i64)(height) << (FTS5_DATA_PAGE_B)) + \ ((i64)(pgno)) \ ) -#if FTS5_MAX_PREFIX_INDEXES > ((1<pConfig; Fts5Structure *pRet = 0; /* Object to return */ Fts5Data *pData; /* %_data entry containing structure record */ int iCookie; /* Configuration cookie */ - assert( iIdx<=pConfig->nPrefix ); - pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID(iIdx)); + pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID); if( !pData ) return 0; p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet); if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); @@ -1196,16 +1193,16 @@ return nSegment; } #endif /* -** Serialize and store the "structure" record for index iIdx. +** Serialize and store the "structure" record. ** ** If an error occurs, leave an error code in the Fts5Index object. If an ** error has already occurred, this function is a no-op. */ -static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){ +static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ if( p->rc==SQLITE_OK ){ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ @@ -1234,11 +1231,11 @@ fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); } } - fts5DataWrite(p, FTS5_STRUCTURE_ROWID(iIdx), buf.p, buf.n); + fts5DataWrite(p, FTS5_STRUCTURE_ROWID, buf.p, buf.n); fts5BufferFree(&buf); } } #if 0 @@ -1530,19 +1527,19 @@ } static Fts5DlidxIter *fts5DlidxIterInit( Fts5Index *p, /* Fts5 Backend to iterate within */ int bRev, /* True for ORDER BY ASC */ - int iIdx, int iSegid, /* Segment iSegid within index iIdx */ + int iSegid, /* Segment id */ int iLeafPg /* Leaf page number to load dlidx for */ ){ Fts5DlidxIter *pIter; pIter = (Fts5DlidxIter*)fts5IdxMalloc(p, sizeof(Fts5DlidxIter)); if( pIter==0 ) return 0; - pIter->pData = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iIdx, iSegid, iLeafPg)); + pIter->pData = fts5DataRead(p, FTS5_DOCLIST_IDX_ROWID(iSegid, iLeafPg)); if( pIter->pData==0 ){ sqlite3_free(pIter); pIter = 0; }else{ pIter->iLeafPgno = iLeafPg; @@ -1581,11 +1578,11 @@ Fts5StructureSegment *pSeg = pIter->pSeg; fts5DataRelease(pIter->pLeaf); pIter->iLeafPgno++; if( pIter->iLeafPgno<=pSeg->pgnoLast ){ pIter->pLeaf = fts5DataRead(p, - FTS5_SEGMENT_ROWID(pIter->iIdx, pSeg->iSegid, 0, pIter->iLeafPgno) + FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pIter->iLeafPgno) ); }else{ pIter->pLeaf = 0; } } @@ -1667,19 +1664,18 @@ pIter->iLeafOffset = iOff; } /* ** Initialize the iterator object pIter to iterate through the entries in -** segment pSeg within index iIdx. The iterator is left pointing to the -** first entry when this function returns. +** segment pSeg. The iterator is left pointing to the first entry when +** this function returns. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterInit( - Fts5Index *p, - int iIdx, /* Config.aHash[] index of FTS index */ + Fts5Index *p, /* FTS index object */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ if( pSeg->pgnoFirst==0 ){ /* This happens if the segment is being used as an input to an incremental @@ -1692,11 +1688,10 @@ } if( p->rc==SQLITE_OK ){ memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; - pIter->iIdx = iIdx; pIter->iLeafPgno = pSeg->pgnoFirst-1; fts5SegIterNextPage(p, pIter); } if( p->rc==SQLITE_OK ){ @@ -1769,11 +1764,11 @@ pIter->pLeaf = 0; while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){ Fts5Data *pNew; pIter->iLeafPgno--; pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID( - pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno + pIter->pSeg->iSegid, 0, pIter->iLeafPgno )); if( pNew ){ if( pIter->iLeafPgno==pIter->iTermLeafPgno ){ if( pIter->iTermLeafOffsetn ){ pIter->pLeaf = pNew; @@ -1877,12 +1872,12 @@ }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm; int nList; if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ - sqlite3Fts5HashScanNext(p->apHash[0]); - sqlite3Fts5HashScanEntry(p->apHash[0], &zTerm, &pList, &nList); + sqlite3Fts5HashScanNext(p->pHash); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); } if( pList==0 ){ fts5DataRelease(pIter->pLeaf); pIter->pLeaf = 0; }else{ @@ -1933,11 +1928,11 @@ /* ** Iterator pIter currently points to the first rowid in a doclist. This ** function sets the iterator up so that iterates in reverse order through ** the doclist. */ -static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){ +static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5DlidxIter *pDlidx = pIter->pDlidx; Fts5Data *pLast = 0; int pgnoLast = 0; if( pDlidx ){ @@ -1944,11 +1939,11 @@ /* If the doclist-iterator is already at EOF, then the current doclist ** contains no entries except those on the current page. */ if( fts5DlidxIterEof(p, pDlidx)==0 ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = pDlidx->iLeafPgno; - pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pgnoLast)); + pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, 0, pgnoLast)); }else{ pIter->iLeafOffset -= sqlite3Fts5GetVarintLen(pIter->nPos*2+pIter->bDel); } }else{ int iOff; /* Byte offset within pLeaf */ @@ -1987,11 +1982,11 @@ Fts5StructureSegment *pSeg = pIter->pSeg; /* The last rowid in the doclist may not be on the current page. Search ** forward to find the page containing the last rowid. */ for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ - i64 iAbs = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pgno); + i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, pgno); Fts5Data *pNew = fts5DataRead(p, iAbs); if( pNew ){ int iRowid, iTerm; fts5LeafHeader(pNew, &iRowid, &iTerm); if( iRowid ){ @@ -2027,17 +2022,16 @@ fts5SegIterReverseInitPage(p, pIter); } /* -** Iterator pIter currently points to the first rowid of a doclist within -** index iIdx. There is a doclist-index associated with the final term on -** the current page. If the current term is the last term on the page, -** load the doclist-index from disk and initialize an iterator at -** (pIter->pDlidx). +** Iterator pIter currently points to the first rowid of a doclist. +** There is a doclist-index associated with the final term on the current +** page. If the current term is the last term on the page, load the +** doclist-index from disk and initialize an iterator at (pIter->pDlidx). */ -static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){ +static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ int iSeg = pIter->pSeg->iSegid; int bRev = (pIter->flags & FTS5_SEGITER_REVERSE); Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ assert( pIter->flags & FTS5_SEGITER_ONETERM ); @@ -2060,45 +2054,42 @@ iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy); iOff += nPos; } } - pIter->pDlidx = fts5DlidxIterInit(p, bRev, iIdx, iSeg, pIter->iTermLeafPgno); + pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } /* ** Initialize the object pIter to point to term pTerm/nTerm within segment -** pSeg, index iIdx. If there is no such term in the index, the iterator -** is set to EOF. +** pSeg. If there is no such term in the index, the iterator is set to EOF. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterSeekInit( Fts5Index *p, /* FTS5 backend */ - int iIdx, /* Config.aHash[] index of FTS index */ const u8 *pTerm, int nTerm, /* Term to seek to */ int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; int h; - int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0); + int bGe = (flags & FTS5INDEX_QUERY_PREFIX); int bDlidx = 0; /* True if there is a doclist-index */ assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; - pIter->iIdx = iIdx; /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ for(h=pSeg->nHeight-1; h>0; h--){ Fts5NodeIter node; /* For iterating through internal nodes */ - i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, h, iPg); + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg); Fts5Data *pNode = fts5DataRead(p, iRowid); if( pNode==0 ) break; fts5NodeIterInit(pNode->p, pNode->n, &node); assert( node.term.n==0 ); @@ -2147,50 +2138,48 @@ if( pIter->pLeaf ){ if( flags & FTS5INDEX_QUERY_DESC ){ pIter->flags |= FTS5_SEGITER_REVERSE; } if( bDlidx ){ - fts5SegIterLoadDlidx(p, iIdx, pIter); + fts5SegIterLoadDlidx(p, pIter); } if( flags & FTS5INDEX_QUERY_DESC ){ - fts5SegIterReverse(p, iIdx, pIter); + fts5SegIterReverse(p, pIter); } } } } /* ** Initialize the object pIter to point to term pTerm/nTerm within the -** in-memory hash table iIdx. If there is no such term in the table, the +** in-memory hash table. If there is no such term in the hash-table, the ** iterator is set to EOF. ** ** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterHashInit( Fts5Index *p, /* FTS5 backend */ - int iIdx, /* Config.aHash[] index of FTS index */ const u8 *pTerm, int nTerm, /* Term to seek to */ int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5SegIter *pIter /* Object to populate */ ){ - Fts5Hash *pHash = p->apHash[iIdx]; const u8 *pList = 0; int nList = 0; const u8 *z = 0; int n = 0; - assert( pHash ); + assert( p->pHash ); assert( p->rc==SQLITE_OK ); - if( pTerm==0 || (iIdx==0 && (flags & FTS5INDEX_QUERY_PREFIX)) ){ - p->rc = sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm); - sqlite3Fts5HashScanEntry(pHash, (const char**)&z, &pList, &nList); + if( pTerm==0 || (flags & FTS5INDEX_QUERY_PREFIX) ){ + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); n = (z ? strlen((const char*)z) : 0); }else{ pIter->flags |= FTS5_SEGITER_ONETERM; - sqlite3Fts5HashQuery(pHash, (const char*)pTerm, nTerm, &pList, &nList); + sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); z = pTerm; n = nTerm; } if( pList ){ @@ -2550,11 +2539,10 @@ ** iterated data. */ static void fts5MultiIterNew( Fts5Index *p, /* FTS5 backend to iterate within */ Fts5Structure *pStruct, /* Structure of specific index */ - int iIdx, /* Config.aHash[] index of FTS index */ int bSkipEmpty, /* True to ignore delete-keys */ int flags, /* FTS5INDEX_QUERY_XXX flags */ const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */ int iLevel, /* Level to iterate (-1 for all) */ int nSegment, /* Number of segments to merge (iLevel>=0) */ @@ -2572,11 +2560,11 @@ /* Allocate space for the new multi-seg-iterator. */ if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->apHash ? 1 : 0); + nSeg += (p->pHash ? 1 : 0); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } for(nSlot=2; nSlotbSkipEmpty = bSkipEmpty; /* Initialize each of the component segment iterators. */ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->apHash ){ + if( p->pHash ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; - fts5SegIterHashInit(p, iIdx, pTerm, nTerm, flags, pIter); + fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); } for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; Fts5SegIter *pIter = &pNew->aSeg[iIter++]; if( pTerm==0 ){ - fts5SegIterInit(p, iIdx, pSeg, pIter); + fts5SegIterInit(p, pSeg, pIter); }else{ - fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, flags, pSeg, pIter); + fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter); } } } }else{ pLvl = &pStruct->aLevel[iLevel]; for(iSeg=nSeg-1; iSeg>=0; iSeg--){ - fts5SegIterInit(p, iIdx, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); + fts5SegIterInit(p, &pLvl->aSeg[iSeg], &pNew->aSeg[iIter++]); } } assert( iIter==nSeg ); /* If the above was successful, each component iterators now points @@ -2732,12 +2720,11 @@ memset(pIter, 0, sizeof(*pIter)); /* If Fts5SegIter.pSeg is NULL, then this iterator iterates through data ** currently stored in a hash table. In this case there is no leaf-rowid ** to calculate. */ if( pSeg->pSeg ){ - int iId = pSeg->pSeg->iSegid; - i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno); + i64 rowid = FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, 0, pSeg->iLeafPgno); pIter->iLeafRowid = rowid; } fts5DataReference(pLeaf); pIter->pLeaf = pLeaf; @@ -2792,17 +2779,13 @@ /* ** Discard all data currently cached in the hash-tables. */ static void fts5IndexDiscardData(Fts5Index *p){ - assert( p->apHash || p->nPendingData==0 ); - if( p->apHash ){ - Fts5Config *pConfig = p->pConfig; - int i; - for(i=0; i<=pConfig->nPrefix; i++){ - sqlite3Fts5HashClear(p->apHash[i]); - } + assert( p->pHash || p->nPendingData==0 ); + if( p->pHash ){ + sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; } } /* @@ -2830,12 +2813,11 @@ int bFlag = 0; Fts5PageWriter *pPg; pPg = &pWriter->aWriter[1]; if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ i64 iKey = FTS5_DOCLIST_IDX_ROWID( - pWriter->iIdx, pWriter->iSegid, - pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty + pWriter->iSegid, pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty ); assert( pWriter->cdlidx.n>0 ); fts5DataWrite(p, iKey, pWriter->cdlidx.p, pWriter->cdlidx.n); bFlag = 1; } @@ -2899,13 +2881,11 @@ fts5WriteBtreeNEmpty(p, pWriter); if( pPage->buf.n>=p->pConfig->pgsz ){ /* pPage will be written to disk. The term will be written into the ** parent of pPage. */ - i64 iRowid = FTS5_SEGMENT_ROWID( - pWriter->iIdx, pWriter->iSegid, iHeight, pPage->pgno - ); + i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno); fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); fts5BufferZero(&pPage->buf); fts5BufferZero(&pPage->term); fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno); pPage->pgno++; @@ -2969,11 +2949,11 @@ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); fts5WriteBtreeNoTerm(p, pWriter); } /* Write the current page to the db. */ - iRowid = FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, 0, pPage->pgno); + iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, 0, pPage->pgno); fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); /* Initialize the next page. */ fts5BufferZero(&pPage->buf); fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); @@ -3177,11 +3157,11 @@ *pnHeight = pWriter->nWriter; for(i=1; inWriter; i++){ Fts5PageWriter *pPg = &pWriter->aWriter[i]; fts5DataWrite(p, - FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pPg->pgno), + FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno), pPg->buf.p, pPg->buf.n ); } } } @@ -3195,14 +3175,13 @@ } static void fts5WriteInit( Fts5Index *p, Fts5SegWriter *pWriter, - int iIdx, int iSegid + int iSegid ){ memset(pWriter, 0, sizeof(Fts5SegWriter)); - pWriter->iIdx = iIdx; pWriter->iSegid = iSegid; pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p,sizeof(Fts5PageWriter)); if( pWriter->aWriter==0 ) return; pWriter->nWriter = 1; @@ -3211,26 +3190,24 @@ } static void fts5WriteInitForAppend( Fts5Index *p, /* FTS5 backend object */ Fts5SegWriter *pWriter, /* Writer to initialize */ - int iIdx, /* Index segment is a part of */ Fts5StructureSegment *pSeg /* Segment object to append to */ ){ int nByte = pSeg->nHeight * sizeof(Fts5PageWriter); memset(pWriter, 0, sizeof(Fts5SegWriter)); - pWriter->iIdx = iIdx; pWriter->iSegid = pSeg->iSegid; pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte); if( p->rc==SQLITE_OK ){ int pgno = 1; int i; pWriter->nWriter = pSeg->nHeight; pWriter->aWriter[0].pgno = pSeg->pgnoLast+1; for(i=pSeg->nHeight-1; i>0; i--){ - i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iIdx, pWriter->iSegid, i, pgno); + i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno); Fts5PageWriter *pPg = &pWriter->aWriter[i]; pPg->pgno = pgno; fts5DataBuffer(p, &pPg->buf, iRowid); if( p->rc==SQLITE_OK ){ Fts5NodeIter ss; @@ -3274,21 +3251,21 @@ i64 iLeafRowid; Fts5Data *pData; int iId = pSeg->pSeg->iSegid; u8 aHdr[4] = {0x00, 0x00, 0x00, 0x04}; - iLeafRowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iTermLeafPgno); + iLeafRowid = FTS5_SEGMENT_ROWID(iId, 0, pSeg->iTermLeafPgno); pData = fts5DataRead(p, iLeafRowid); if( pData ){ fts5BufferZero(&buf); fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; - fts5DataDelete(p, FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, 1),iLeafRowid); + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid); fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } } fts5BufferFree(&buf); @@ -3297,12 +3274,11 @@ /* ** */ static void fts5IndexMergeLevel( Fts5Index *p, /* FTS5 backend object */ - int iIdx, /* Index to work on */ - Fts5Structure **ppStruct, /* IN/OUT: Stucture of index iIdx */ + Fts5Structure **ppStruct, /* IN/OUT: Stucture of index */ int iLvl, /* Level to read input from */ int *pnRem /* Write up to this many output leaves */ ){ Fts5Structure *pStruct = *ppStruct; Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; @@ -3319,16 +3295,15 @@ assert( iLvlnLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); - writer.iIdx = iIdx; if( pLvl->nMerge ){ pLvlOut = &pStruct->aLevel[iLvl+1]; assert( pLvlOut->nSeg>0 ); nInput = pLvl->nMerge; - fts5WriteInitForAppend(p, &writer, iIdx, &pLvlOut->aSeg[pLvlOut->nSeg-1]); + fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]); pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; }else{ int iSegid = fts5AllocateSegid(p, pStruct); /* Extend the Fts5Structure object as required to ensure the output @@ -3340,11 +3315,11 @@ fts5StructureExtendLevel(&p->rc, pStruct, iLvl+1, 1, 0); if( p->rc ) return; pLvl = &pStruct->aLevel[iLvl]; pLvlOut = &pStruct->aLevel[iLvl+1]; - fts5WriteInit(p, &writer, iIdx, iSegid); + fts5WriteInit(p, &writer, iSegid); /* Add the new segment to the output level */ pSeg = &pLvlOut->aSeg[pLvlOut->nSeg]; pLvlOut->nSeg++; pSeg->pgnoFirst = 1; @@ -3360,11 +3335,11 @@ fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl); fflush(stdout); #endif assert( iLvl>=0 ); - for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter); + for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; Fts5ChunkIter sPos; /* Used to iterate through position list */ @@ -3412,11 +3387,11 @@ if( fts5MultiIterEof(p, pIter) ){ int i; /* Remove the redundant segments from the %_data table */ for(i=0; iaSeg[i].iSegid); + fts5DataRemoveSegment(p, pLvl->aSeg[i].iSegid); } /* Remove the redundant segments from the input level */ if( pLvl->nSeg!=nInput ){ int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment); @@ -3439,15 +3414,14 @@ fts5BufferFree(&term); if( pnRem ) *pnRem -= writer.nLeafWritten; } /* -** Do up to nPg pages of automerge work on index iIdx. +** Do up to nPg pages of automerge work on the index. */ static void fts5IndexMerge( Fts5Index *p, /* FTS5 backend object */ - int iIdx, /* Index to work on */ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ int nPg /* Pages of work to do */ ){ int nRem = nPg; Fts5Structure *pStruct = *ppStruct; @@ -3483,30 +3457,28 @@ if( nBestpConfig->nAutomerge && pStruct->aLevel[iBestLvl].nMerge==0 ){ break; } - fts5IndexMergeLevel(p, iIdx, &pStruct, iBestLvl, &nRem); + fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } } *ppStruct = pStruct; } /* ** A total of nLeaf leaf pages of data has just been flushed to a level-0 -** segments in index iIdx with structure pStruct. This function updates the -** write-counter accordingly and, if necessary, performs incremental merge -** work. +** segment. This function updates the write-counter accordingly and, if +** necessary, performs incremental merge work. ** ** If an error occurs, set the Fts5Index.rc error code. If an error has ** already occurred, this function is a no-op. */ static void fts5IndexAutomerge( Fts5Index *p, /* FTS5 backend object */ - int iIdx, /* Index to work on */ Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ int nLeaf /* Number of output leaves just written */ ){ if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){ Fts5Structure *pStruct = *ppStruct; @@ -3518,26 +3490,25 @@ nWrite = pStruct->nWriteCounter; nWork = ((nWrite + nLeaf) / p->nWorkUnit) - (nWrite / p->nWorkUnit); pStruct->nWriteCounter += nLeaf; nRem = p->nWorkUnit * nWork * pStruct->nLevel; - fts5IndexMerge(p, iIdx, ppStruct, nRem); + fts5IndexMerge(p, ppStruct, nRem); } } static void fts5IndexCrisismerge( Fts5Index *p, /* FTS5 backend object */ - int iIdx, /* Index to work on */ Fts5Structure **ppStruct /* IN/OUT: Current structure of index */ ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; int iLvl = 0; assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, iIdx, &pStruct, iLvl, 0); + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); fts5StructurePromote(p, iLvl+1, pStruct); iLvl++; } *ppStruct = pStruct; } @@ -3582,19 +3553,19 @@ ** segment on disk. Also update the corresponding structure record. ** ** If an error occurs, set the Fts5Index.rc error code. If an error has ** already occurred, this function is a no-op. */ -static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){ - Fts5Hash *pHash = p->apHash[iHash]; +static void fts5FlushOneHash(Fts5Index *p){ + Fts5Hash *pHash = p->pHash; Fts5Structure *pStruct; int iSegid; int pgnoLast = 0; /* Last leaf page number in segment */ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ - pStruct = fts5StructureRead(p, iHash); + pStruct = fts5StructureRead(p); iSegid = fts5AllocateSegid(p, pStruct); if( iSegid ){ const int pgsz = p->pConfig->pgsz; @@ -3602,11 +3573,11 @@ int nHeight; /* Height of new segment b-tree */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ const u8 *zPrev = 0; Fts5SegWriter writer; - fts5WriteInit(p, &writer, iHash, iSegid); + fts5WriteInit(p, &writer, iSegid); /* Pre-allocate the buffer used to assemble leaf pages to the target ** page size. */ assert( pgsz>0 ); pBuf = &writer.aWriter[0].buf; @@ -3747,101 +3718,91 @@ } fts5StructurePromote(p, 0, pStruct); } - fts5IndexAutomerge(p, iHash, &pStruct, pgnoLast); - fts5IndexCrisismerge(p, iHash, &pStruct); - fts5StructureWrite(p, iHash, pStruct); + fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexCrisismerge(p, &pStruct); + fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); } /* ** Flush any data stored in the in-memory hash tables to the database. */ static void fts5IndexFlush(Fts5Index *p){ - Fts5Config *pConfig = p->pConfig; - int i; /* Used to iterate through indexes */ - int nLeaf = 0; /* Number of leaves written */ - - /* If an error has already occured this call is a no-op. */ - if( p->nPendingData==0 ) return; - assert( p->apHash ); - - /* Flush the terms and each prefix index to disk */ - for(i=0; i<=pConfig->nPrefix; i++){ - fts5FlushOneHash(p, i, &nLeaf); + /* Unless it is empty, flush the hash table to disk */ + if( p->rc==SQLITE_OK && p->nPendingData ){ + assert( p->pHash ); + p->nPendingData = 0; + fts5FlushOneHash(p); } - p->nPendingData = 0; } int sqlite3Fts5IndexOptimize(Fts5Index *p){ - Fts5Config *pConfig = p->pConfig; - int i; + Fts5Structure *pStruct; + Fts5Structure *pNew = 0; + int nSeg = 0; assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); - for(i=0; i<=pConfig->nPrefix; i++){ - Fts5Structure *pStruct = fts5StructureRead(p, i); - Fts5Structure *pNew = 0; - int nSeg = 0; - if( pStruct ){ - assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); - nSeg = pStruct->nSegment; - if( nSeg>1 ){ - int nByte = sizeof(Fts5Structure); - nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); - pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); - } - } - if( pNew ){ - Fts5StructureLevel *pLvl; - int nByte = nSeg * sizeof(Fts5StructureSegment); - pNew->nLevel = pStruct->nLevel+1; - pNew->nWriteCounter = pStruct->nWriteCounter; - pLvl = &pNew->aLevel[pStruct->nLevel]; - pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); - if( pLvl->aSeg ){ - int iLvl, iSeg; - int iSegOut = 0; - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; - iSegOut++; - } + pStruct = fts5StructureRead(p); + + if( pStruct ){ + assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); + nSeg = pStruct->nSegment; + if( nSeg>1 ){ + int nByte = sizeof(Fts5Structure); + nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel); + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(&p->rc, nByte); + } + } + if( pNew ){ + Fts5StructureLevel *pLvl; + int nByte = nSeg * sizeof(Fts5StructureSegment); + pNew->nLevel = pStruct->nLevel+1; + pNew->nWriteCounter = pStruct->nWriteCounter; + pLvl = &pNew->aLevel[pStruct->nLevel]; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pLvl->aSeg ){ + int iLvl, iSeg; + int iSegOut = 0; + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + pLvl->aSeg[iSegOut] = pStruct->aLevel[iLvl].aSeg[iSeg]; + iSegOut++; } - pNew->nSegment = pLvl->nSeg = nSeg; - }else{ - sqlite3_free(pNew); - pNew = 0; - } - } - - if( pNew ){ - int iLvl = pNew->nLevel-1; - while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ - int nRem = FTS5_OPT_WORK_UNIT; - fts5IndexMergeLevel(p, i, &pNew, iLvl, &nRem); - } - - fts5StructureWrite(p, i, pNew); - fts5StructureRelease(pNew); - } - - fts5StructureRelease(pStruct); - } - + } + pNew->nSegment = pLvl->nSeg = nSeg; + }else{ + sqlite3_free(pNew); + pNew = 0; + } + } + + if( pNew ){ + int iLvl = pNew->nLevel-1; + while( p->rc==SQLITE_OK && pNew->aLevel[iLvl].nSeg>0 ){ + int nRem = FTS5_OPT_WORK_UNIT; + fts5IndexMergeLevel(p, &pNew, iLvl, &nRem); + } + + fts5StructureWrite(p, pNew); + fts5StructureRelease(pNew); + } + + fts5StructureRelease(pStruct); return fts5IndexReturn(p); } int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pStruct; - pStruct = fts5StructureRead(p, 0); - fts5IndexMerge(p, 0, &pStruct, nMerge); - fts5StructureWrite(p, 0, pStruct); + pStruct = fts5StructureRead(p); + fts5IndexMerge(p, &pStruct, nMerge); + fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); return fts5IndexReturn(p); } @@ -4038,21 +3999,21 @@ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); - pStruct = fts5StructureRead(p, 0); + pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ Fts5DoclistIter *pDoclist; int i; i64 iLastRowid = 0; Fts5MultiSegIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Buffer doclist; memset(&doclist, 0, sizeof(doclist)); - for(fts5MultiIterNew(p, pStruct, 0, 1, 1, pToken, nToken, -1, 0, &p1); + for(fts5MultiIterNew(p, pStruct, 1, 1, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; fts5MultiIterNext(p, p1, 0, 0) ){ i64 iRowid = fts5MultiIterRowid(p1); int nTerm; @@ -4111,36 +4072,16 @@ ** to the document with rowid iRowid. */ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ assert( p->rc==SQLITE_OK ); - /* Allocate hash tables if they have not already been allocated */ - if( p->apHash==0 ){ - int i; - int rc = SQLITE_OK; - int nHash = p->pConfig->nPrefix + 1; - Fts5Hash **apNew; - - apNew = (Fts5Hash**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Hash*)*nHash); - for(i=0; rc==SQLITE_OK && inPendingData); - } - if( rc==SQLITE_OK ){ - p->apHash = apNew; - }else{ - if( apNew ){ - for(i=0; ipHash==0 ){ + p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); } if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ - assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); } p->iWriteRowid = iRowid; return fts5IndexReturn(p); } @@ -4172,20 +4113,17 @@ ** The %_data table is completely empty when this function is called. This ** function populates it with the initial structure objects for each index, ** and the initial version of the "averages" record (a zero-byte blob). */ int sqlite3Fts5IndexReinit(Fts5Index *p){ - int i; Fts5Structure s; + assert( p->rc==SQLITE_OK ); + p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); + memset(&s, 0, sizeof(Fts5Structure)); - for(i=0; ipConfig->nPrefix+1; i++){ - fts5StructureWrite(p, i, &s); - } - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0); - } + fts5StructureWrite(p, &s); return fts5IndexReturn(p); } /* @@ -4237,17 +4175,12 @@ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); - if( p->apHash ){ - int i; - for(i=0; i<=p->pConfig->nPrefix; i++){ - sqlite3Fts5HashFree(p->apHash[i]); - } - sqlite3_free(p->apHash); - } + sqlite3Fts5HashFree(p->pHash); + sqlite3Fts5BufferFree(&p->scratch); sqlite3_free(p->zDataTbl); sqlite3_free(p); } return rc; } @@ -4300,25 +4233,25 @@ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ int i; /* Used to iterate through indexes */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ Fts5Config *pConfig = p->pConfig; assert( p->rc==SQLITE_OK ); - /* Add the new token to the main terms hash table. And to each of the - ** prefix hash tables that it is large enough for. */ + /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( - p->apHash[0], p->iWriteRowid, iCol, iPos, pToken, nToken + p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); + for(i=0; inPrefix && rc==SQLITE_OK; i++){ int nByte = fts5IndexCharlenToBytelen(pToken, nToken, pConfig->aPrefix[i]); if( nByte ){ - rc = sqlite3Fts5HashWrite( - p->apHash[i+1], p->iWriteRowid, iCol, iPos, pToken, nByte + rc = sqlite3Fts5HashWrite(p->pHash, + p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX+i+1, pToken, nByte ); } } return rc; @@ -4335,10 +4268,15 @@ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5IndexIter *pRet; int iIdx = 0; + Fts5Buffer buf = {0, 0, 0}; + + if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ + memcpy(&buf.p[1], pToken, nToken); + } if( flags & FTS5INDEX_QUERY_PREFIX ){ if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ iIdx = 1+pConfig->nPrefix; }else{ @@ -4353,27 +4291,31 @@ if( pRet ){ memset(pRet, 0, sizeof(Fts5IndexIter)); pRet->pIndex = p; if( iIdx<=pConfig->nPrefix ){ - pRet->pStruct = fts5StructureRead(p, iIdx); + buf.p[0] = FTS5_MAIN_PREFIX + iIdx; + pRet->pStruct = fts5StructureRead(p); if( pRet->pStruct ){ - fts5MultiIterNew(p, pRet->pStruct, - iIdx, 1, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti + int f = (flags & ~FTS5INDEX_QUERY_PREFIX); + fts5MultiIterNew( + p, pRet->pStruct, 1, f, buf.p, nToken+1, -1, 0, &pRet->pMulti ); } }else{ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; - fts5SetupPrefixIter(p, bDesc, (const u8*)pToken, nToken, pRet); + buf.p[0] = FTS5_MAIN_PREFIX; + fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pRet); } } if( p->rc ){ sqlite3Fts5IterClose(pRet); pRet = 0; } *ppIter = pRet; + sqlite3Fts5BufferFree(&buf); return fts5IndexReturn(p); } /* ** Return true if the iterator passed as the only argument is at EOF. @@ -4518,35 +4460,32 @@ ** ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ - int rc = SQLITE_OK; - Fts5Config *pConfig = p->pConfig; - u8 aCookie[4]; - int i; + int rc; /* Return code */ + Fts5Config *pConfig = p->pConfig; /* Configuration object */ + u8 aCookie[4]; /* Binary representation of iNew */ assert( p->rc==SQLITE_OK ); + sqlite3Fts5Put32(aCookie, iNew); - for(i=0; rc==SQLITE_OK && i<=pConfig->nPrefix; i++){ - sqlite3_blob *pBlob = 0; - i64 iRowid = FTS5_STRUCTURE_ROWID(i); - rc = sqlite3_blob_open( - pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 1, &pBlob - ); - if( rc==SQLITE_OK ){ - sqlite3_blob_write(pBlob, aCookie, 4, 0); - rc = sqlite3_blob_close(pBlob); - } + sqlite3_blob *pBlob = 0; + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, + "block", FTS5_STRUCTURE_ROWID, 1, &pBlob + ); + if( rc==SQLITE_OK ){ + sqlite3_blob_write(pBlob, aCookie, 4, 0); + rc = sqlite3_blob_close(pBlob); } return rc; } int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ Fts5Structure *pStruct; - pStruct = fts5StructureRead(p, 0); + pStruct = fts5StructureRead(p); fts5StructureRelease(pStruct); return fts5IndexReturn(p); } @@ -4561,24 +4500,25 @@ */ static u64 fts5IndexEntryCksum( i64 iRowid, int iCol, int iPos, - const char *pTerm, + int iIdx, + const char *pTerm, int nTerm ){ int i; u64 ret = iRowid; ret += (ret<<3) + iCol; ret += (ret<<3) + iPos; + if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); for(i=0; iaLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte); } if( p->rc==SQLITE_OK ){ pIter->nLvl = pSeg->nHeight-1; - pIter->iIdx = iIdx; pIter->p = p; pIter->pSeg = pSeg; } for(i=0; p->rc==SQLITE_OK && inLvl; i++){ - i64 iRowid = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, i+1, 1); + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1); Fts5Data *pData; pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid); if( pData ){ fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s); } @@ -4633,11 +4572,11 @@ pIter->bEof = 1; }else{ int iSegid = pIter->pSeg->iSegid; for(i--; i>=0; i--){ Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - i64 iRowid = FTS5_SEGMENT_ROWID(pIter->iIdx,iSegid,i+1,pLvl[1].s.iChild); + i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild); pLvl->pData = fts5DataRead(p, iRowid); if( pLvl->pData ){ fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s); } } @@ -4666,24 +4605,23 @@ ** This function is purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. ** ** Instead, it tests that the same set of pgno/rowid combinations are ** visited regardless of whether the doclist-index identified by parameters -** iIdx/iSegid/iLeaf is iterated in forwards or reverse order. +** iSegid/iLeaf is iterated in forwards or reverse order. */ #ifdef SQLITE_DEBUG static void fts5DlidxIterTestReverse( Fts5Index *p, - int iIdx, /* Index to load doclist-index from */ int iSegid, /* Segment id to load from */ int iLeaf /* Load doclist-index for this leaf */ ){ Fts5DlidxIter *pDlidx = 0; i64 cksum1 = 13; i64 cksum2 = 13; - for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iLeaf); + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iLeaf); fts5DlidxIterEof(p, pDlidx)==0; fts5DlidxIterNext(pDlidx) ){ assert( pDlidx->iLeafPgno>iLeaf ); cksum1 = (cksum1 ^ ( (i64)(pDlidx->iLeafPgno) << 32 )); @@ -4690,11 +4628,11 @@ cksum1 = (cksum1 ^ pDlidx->iRowid); } fts5DlidxIterFree(pDlidx); pDlidx = 0; - for(pDlidx=fts5DlidxIterInit(p, 1, iIdx, iSegid, iLeaf); + for(pDlidx=fts5DlidxIterInit(p, 1, iSegid, iLeaf); fts5DlidxIterEof(p, pDlidx)==0; fts5DlidxIterPrev(pDlidx) ){ assert( pDlidx->iLeafPgno>iLeaf ); cksum2 = (cksum2 ^ ( (i64)(pDlidx->iLeafPgno) << 32 )); @@ -4704,24 +4642,23 @@ pDlidx = 0; if( p->rc==SQLITE_OK && cksum1!=cksum2 ) p->rc = FTS5_CORRUPT; } #else -# define fts5DlidxIterTestReverse(w,x,y,z) +# define fts5DlidxIterTestReverse(x,y,z) #endif static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ - int iIdx, /* Index that pSeg is a part of */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ if( pSeg->pgnoFirst==0 ) return; /* Iterate through the b-tree hierarchy. */ - for(fts5BtreeIterInit(p, iIdx, pSeg, &iter); + for(fts5BtreeIterInit(p, pSeg, &iter); p->rc==SQLITE_OK && iter.bEof==0; fts5BtreeIterNext(&iter) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ @@ -4729,11 +4666,11 @@ int i; /* Used to iterate through empty leaves */ /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ if( iter.iLeafpgnoFirst ) continue; - iRow = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, iter.iLeaf); + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf); pLeaf = fts5DataRead(p, iRow); if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal ** to or larger than the split-key in iter.term. */ @@ -4769,18 +4706,18 @@ int iPrevLeaf = iter.iLeaf; int iSegid = pSeg->iSegid; int iPg; i64 iKey; - for(pDlidx=fts5DlidxIterInit(p, 0, iIdx, iSegid, iter.iLeaf); + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf); fts5DlidxIterEof(p, pDlidx)==0; fts5DlidxIterNext(pDlidx) ){ /* Check any rowid-less pages that occur before the current leaf. */ for(iPg=iPrevLeaf+1; iPgiLeafPgno; iPg++){ - iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg); + iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; fts5DataRelease(pLeaf); } @@ -4787,11 +4724,11 @@ } iPrevLeaf = pDlidx->iLeafPgno; /* Check that the leaf page indicated by the iterator really does ** contain the rowid suggested by the same. */ - iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, pDlidx->iLeafPgno); + iKey = FTS5_SEGMENT_ROWID(iSegid, 0, pDlidx->iLeafPgno); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ i64 iRowid; int iRowidOff = fts5GetU16(&pLeaf->p[0]); getVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); @@ -4800,20 +4737,20 @@ } } for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){ - iKey = FTS5_SEGMENT_ROWID(iIdx, iSegid, 0, iPg); + iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); pLeaf = fts5DataRead(p, iKey); if( pLeaf ){ if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; fts5DataRelease(pLeaf); } } fts5DlidxIterFree(pDlidx); - fts5DlidxIterTestReverse(p, iIdx, iSegid, iter.iLeaf); + fts5DlidxIterTestReverse(p, iSegid, iter.iLeaf); } } /* Either iter.iLeaf must be the rightmost leaf-page in the segment, or ** else the segment has been completely emptied by an ongoing merge @@ -4828,15 +4765,16 @@ fts5BtreeIterFree(&iter); } static int fts5QueryCksum( - Fts5Index *p, - const char *z, - int n, - int flags, - u64 *pCksum + Fts5Index *p, /* Fts5 index object */ + int iIdx, + const char *z, /* Index key to query for */ + int n, /* Size of index key in bytes */ + int flags, /* Flags for Fts5IndexQuery */ + u64 *pCksum /* IN/OUT: Checksum value */ ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter); @@ -4851,11 +4789,11 @@ sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); - cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, z, n); + cksum ^= fts5IndexEntryCksum(rowid, iCol, iOff, iIdx, z, n); } rc = sqlite3Fts5IterNext(pIdxIter); } } sqlite3Fts5IterClose(pIdxIter); @@ -4873,30 +4811,29 @@ ** checksum does not match. Return SQLITE_OK if all checks pass without ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ - Fts5Config *pConfig = p->pConfig; - int iIdx; /* Used to iterate through indexes */ u64 cksum2 = 0; /* Checksum based on contents of indexes */ u64 cksum3 = 0; /* Checksum based on contents of indexes */ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ + Fts5MultiSegIter *pIter; /* Used to iterate through entire index */ + Fts5Structure *pStruct; /* Index structure */ + + /* Load the FTS index structure */ + pStruct = fts5StructureRead(p); /* Check that the internal nodes of each segment match the leaves */ - for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){ - Fts5Structure *pStruct = fts5StructureRead(p, iIdx); - if( pStruct ){ - int iLvl, iSeg; - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; - fts5IndexIntegrityCheckSegment(p, iIdx, pSeg); - } + if( pStruct ){ + int iLvl, iSeg; + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + fts5IndexIntegrityCheckSegment(p, pSeg); } } - fts5StructureRelease(pStruct); } /* The cksum argument passed to this function is a checksum calculated ** based on all expected entries in the FTS index (including prefix index ** entries). This block checks that a checksum calculated based on the @@ -4908,72 +4845,73 @@ ** ** As each term visited by the linear scans, a separate query for the ** same term is performed. cksum3 is calculated based on the entries ** extracted by these queries. */ - for(iIdx=0; iIdx<=pConfig->nPrefix; iIdx++){ - Fts5MultiSegIter *pIter; - Fts5Structure *pStruct = fts5StructureRead(p, iIdx); - for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, -1, 0, &pIter); - fts5MultiIterEof(p, pIter)==0; - fts5MultiIterNext(p, pIter, 0, 0) - ){ - int n; /* Size of term in bytes */ - i64 iPos = 0; /* Position read from poslist */ - int iOff = 0; /* Offset within poslist */ - i64 iRowid = fts5MultiIterRowid(pIter); - char *z = (char*)fts5MultiIterTerm(pIter, &n); - - poslist.n = 0; - fts5MultiIterPoslist(p, pIter, 0, &poslist); - while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ - int iCol = FTS5_POS2COLUMN(iPos); - int iTokOff = FTS5_POS2OFFSET(iPos); - cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n); + for(fts5MultiIterNew(p, pStruct, 0, 0, 0, 0, -1, 0, &pIter); + fts5MultiIterEof(p, pIter)==0; + fts5MultiIterNext(p, pIter, 0, 0) + ){ + int n; /* Size of term in bytes */ + i64 iPos = 0; /* Position read from poslist */ + int iOff = 0; /* Offset within poslist */ + i64 iRowid = fts5MultiIterRowid(pIter); + char *z = (char*)fts5MultiIterTerm(pIter, &n); + + poslist.n = 0; + fts5MultiIterPoslist(p, pIter, 0, &poslist); + while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ + int iCol = FTS5_POS2COLUMN(iPos); + int iTokOff = FTS5_POS2OFFSET(iPos); + cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); + } + + /* If this is a new term, query for it. Update cksum3 with the results. */ + if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){ + const char *zTerm = &z[1]; /* The term without the prefix-byte */ + int nTerm = n-1; /* Size of zTerm in bytes */ + int iIdx = (z[0] - FTS5_MAIN_PREFIX); + int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); + int rc; + u64 ck1 = 0; + u64 ck2 = 0; + + /* Check that the results returned for ASC and DESC queries are + ** the same. If not, call this corruption. */ + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, flags, &ck1); + if( rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_DESC; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + } + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + + /* If this is a prefix query, check that the results returned if the + ** the index is disabled are the same. In both ASC and DESC order. */ + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; + } + if( iIdx>0 && rc==SQLITE_OK ){ + int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; + ck2 = 0; + rc = fts5QueryCksum(p, iIdx, zTerm, nTerm, f, &ck2); + if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; } - /* If this is a new term, query for it. Update cksum3 with the results. */ - if( p->rc==SQLITE_OK && (term.n!=n || memcmp(term.p, z, n)) ){ - int rc; - int flags = (iIdx==0 ? 0 : FTS5INDEX_QUERY_PREFIX); - u64 ck1 = 0; - u64 ck2 = 0; - - /* Check that the results returned for ASC and DESC queries are - ** the same. If not, call this corruption. */ - rc = fts5QueryCksum(p, z, n, flags, &ck1); - if( rc==SQLITE_OK ){ - rc = fts5QueryCksum(p, z, n, flags|FTS5INDEX_QUERY_DESC, &ck2); - } - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; - - /* If this is a prefix query, check that the results returned if the - ** the index is disabled are the same. In both ASC and DESC order. */ - if( iIdx>0 && rc==SQLITE_OK ){ - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; - ck2 = 0; - rc = fts5QueryCksum(p, z, n, f, &ck2); - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; - } - if( iIdx>0 && rc==SQLITE_OK ){ - int f = flags|FTS5INDEX_QUERY_TEST_NOIDX|FTS5INDEX_QUERY_DESC; - ck2 = 0; - rc = fts5QueryCksum(p, z, n, f, &ck2); - if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; - } - - cksum3 ^= ck1; - fts5BufferSet(&rc, &term, n, (const u8*)z); - p->rc = rc; - } + cksum3 ^= ck1; + fts5BufferSet(&rc, &term, n, (const u8*)z); + p->rc = rc; } - fts5MultiIterFree(p, pIter); - fts5StructureRelease(pStruct); } + fts5MultiIterFree(p, pIter); + if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT; + fts5StructureRelease(pStruct); fts5BufferFree(&term); fts5BufferFree(&poslist); return fts5IndexReturn(p); } @@ -4991,16 +4929,16 @@ const char *pTerm, int nTerm /* Term at iPos */ ){ u64 ret = 0; /* Return value */ int iIdx; /* For iterating through indexes */ - ret = fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nTerm); + ret = fts5IndexEntryCksum(iRowid, iCol, iPos, 0, pTerm, nTerm); for(iIdx=0; iIdxnPrefix; iIdx++){ int nByte = fts5IndexCharlenToBytelen(pTerm, nTerm, pConfig->aPrefix[iIdx]); if( nByte ){ - ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, pTerm, nByte); + ret ^= fts5IndexEntryCksum(iRowid, iCol, iPos, iIdx+1, pTerm, nByte); } } return ret; } @@ -5015,11 +4953,10 @@ ** Decode a segment-data rowid from the %_data table. This function is ** the opposite of macro FTS5_SEGMENT_ROWID(). */ static void fts5DecodeRowid( i64 iRowid, /* Rowid from %_data table */ - int *piIdx, /* OUT: Index */ int *piSegid, /* OUT: Segment id */ int *piHeight, /* OUT: Height */ int *piPgno /* OUT: Page number */ ){ *piPgno = (int)(iRowid & (((i64)1 << FTS5_DATA_PAGE_B) - 1)); @@ -5027,18 +4964,15 @@ *piHeight = (int)(iRowid & (((i64)1 << FTS5_DATA_HEIGHT_B) - 1)); iRowid >>= FTS5_DATA_HEIGHT_B; *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); - iRowid >>= FTS5_DATA_ID_B; - - *piIdx = (int)(iRowid & (((i64)1 << FTS5_DATA_IDX_B) - 1)); } static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iIdx,iSegid,iHeight,iPgno; /* Rowid compenents */ - fts5DecodeRowid(iKey, &iIdx, &iSegid, &iHeight, &iPgno); + int iSegid, iHeight, iPgno; /* Rowid compenents */ + fts5DecodeRowid(iKey, &iSegid, &iHeight, &iPgno); if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(averages) "); }else{ @@ -5046,16 +4980,16 @@ "{structure idx=%d}", (int)(iKey-10) ); } } else if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx idx=%d segid=%d pgno=%d)", - iIdx, iSegid, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(dlidx segid=%d pgno=%d)", + iSegid, iPgno ); }else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(idx=%d segid=%d h=%d pgno=%d)", - iIdx, iSegid, iHeight, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "(segid=%d h=%d pgno=%d)", + iSegid, iHeight, iPgno ); } } static void fts5DebugStructure( @@ -5161,11 +5095,11 @@ sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args (always 2) */ sqlite3_value **apVal /* Function arguments */ ){ i64 iRowid; /* Rowid for record being decoded */ - int iIdx,iSegid,iHeight,iPgno; /* Rowid components */ + int iSegid,iHeight,iPgno; /* Rowid components */ const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ int rc = SQLITE_OK; /* Return code */ int nSpace = 0; @@ -5178,11 +5112,11 @@ nSpace = n + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; memcpy(a, aBlob, n); - fts5DecodeRowid(iRowid, &iIdx, &iSegid, &iHeight, &iPgno); + fts5DecodeRowid(iRowid, &iSegid, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( iHeight==FTS5_SEGMENT_MAX_HEIGHT ){ Fts5Data dlidx; Fts5DlidxIter iter; @@ -5299,23 +5233,23 @@ sqlite3_result_error(pCtx, "should be: fts5_rowid(subject, ....)", -1); }else{ zArg = (const char*)sqlite3_value_text(apVal[0]); if( 0==sqlite3_stricmp(zArg, "segment") ){ i64 iRowid; - int idx, segid, height, pgno; - if( nArg!=5 ){ + int segid, height, pgno; + if( nArg!=4 ){ sqlite3_result_error(pCtx, - "should be: fts5_rowid('segment', idx, segid, height, pgno))", -1 + "should be: fts5_rowid('segment', segid, height, pgno))", -1 ); }else{ - idx = sqlite3_value_int(apVal[1]); - segid = sqlite3_value_int(apVal[2]); - height = sqlite3_value_int(apVal[3]); - pgno = sqlite3_value_int(apVal[4]); - iRowid = FTS5_SEGMENT_ROWID(idx, segid, height, pgno); + segid = sqlite3_value_int(apVal[1]); + height = sqlite3_value_int(apVal[2]); + pgno = sqlite3_value_int(apVal[3]); + iRowid = FTS5_SEGMENT_ROWID(segid, height, pgno); sqlite3_result_int64(pCtx, iRowid); } +#if 0 }else if( 0==sqlite3_stricmp(zArg, "start-of-index") ){ i64 iRowid; int idx; if( nArg!=2 ){ sqlite3_result_error(pCtx, @@ -5324,10 +5258,11 @@ }else{ idx = sqlite3_value_int(apVal[1]); iRowid = FTS5_SEGMENT_ROWID(idx, 1, 0, 0); sqlite3_result_int64(pCtx, iRowid); } +#endif }else { sqlite3_result_error(pCtx, "first arg to fts5_rowid() must be 'segment' " "or 'start-of-index'" , -1 Index: ext/fts5/fts5_storage.c ================================================================== --- ext/fts5/fts5_storage.c +++ ext/fts5/fts5_storage.c @@ -255,10 +255,13 @@ if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( pConfig, "config", "k PRIMARY KEY, v", 1, pzErr ); } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } } if( rc ){ sqlite3Fts5StorageClose(p); *pp = 0; @@ -541,10 +544,13 @@ /* Reinitialize the %_data table. This call creates the initial structure ** and averages records. */ if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p->pIndex); } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageConfigValue(p, "version", 0, FTS5_CURRENT_VERSION); + } return rc; } int sqlite3Fts5StorageRebuild(Fts5Storage *p){ Fts5Buffer buf = {0,0,0}; @@ -981,22 +987,27 @@ return sqlite3Fts5IndexRollback(p->pIndex); } int sqlite3Fts5StorageConfigValue( Fts5Storage *p, - const char *z, - sqlite3_value *pVal + const char *z, + sqlite3_value *pVal, + int iVal ){ sqlite3_stmt *pReplace = 0; int rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_CONFIG, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_text(pReplace, 1, z, -1, SQLITE_STATIC); - sqlite3_bind_value(pReplace, 2, pVal); + if( pVal ){ + sqlite3_bind_value(pReplace, 2, pVal); + }else{ + sqlite3_bind_int(pReplace, 2, iVal); + } sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pVal ){ int iNew = p->pConfig->iCookie + 1; rc = sqlite3Fts5IndexSetCookie(p->pIndex, iNew); if( rc==SQLITE_OK ){ p->pConfig->iCookie = iNew; } Index: ext/fts5/test/fts5al.test ================================================================== --- ext/fts5/test/fts5al.test +++ ext/fts5/test/fts5al.test @@ -24,21 +24,21 @@ } do_execsql_test 1.1 { CREATE VIRTUAL TABLE ft1 USING fts5(x); SELECT * FROM ft1_config; -} {} +} {version 1} do_execsql_test 1.2 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 32); SELECT * FROM ft1_config; -} {pgsz 32} +} {pgsz 32 version 1} do_execsql_test 1.3 { INSERT INTO ft1(ft1, rank) VALUES('pgsz', 64); SELECT * FROM ft1_config; -} {pgsz 64} +} {pgsz 64 version 1} #-------------------------------------------------------------------------- # Test the logic for parsing the rank() function definition. # foreach {tn defn} { Index: ext/fts5/test/fts5corrupt.test ================================================================== --- ext/fts5/test/fts5corrupt.test +++ ext/fts5/test/fts5corrupt.test @@ -35,20 +35,20 @@ do_execsql_test 1.2 { INSERT INTO t1(t1) VALUES('integrity-check') } set segid [lindex [fts5_level_segids t1] 0] do_test 1.3 { execsql { - DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', 0, $segid, 0, 4); + DELETE FROM t1_data WHERE rowid = fts5_rowid('segment', $segid, 0, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} do_test 1.4 { db_restore_and_reopen execsql { UPDATE t1_data set block = X'00000000' || substr(block, 5) WHERE - rowid = fts5_rowid('segment', 0, $segid, 0, 4); + rowid = fts5_rowid('segment', $segid, 0, 4); } catchsql { INSERT INTO t1(t1) VALUES('integrity-check') } } {1 {database disk image is malformed}} db_restore_and_reopen Index: ext/fts5/test/fts5corrupt2.test ================================================================== --- ext/fts5/test/fts5corrupt2.test +++ ext/fts5/test/fts5corrupt2.test @@ -124,12 +124,12 @@ INSERT INTO x3 SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii; } foreach {tn hdr} { - 1 "\00\00\00\00" - 2 "\FF\FF\FF\FF" + 1 "\x00\x00\x00\x00" + 2 "\xFF\xFF\xFF\xFF" } { set tn2 0 set nCorrupt 0 foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] { if {$rowid & $mask} continue Index: ext/fts5/test/fts5ea.test ================================================================== --- ext/fts5/test/fts5ea.test +++ ext/fts5/test/fts5ea.test @@ -76,10 +76,17 @@ 12 {NEAR(a b, xyz)} {expected integer, got "xyz"} 13 {NEAR(a b, // )} {expected integer, got "//"} } { do_catchsql_test 3.$tn {SELECT fts5_expr($expr, 'name', 'addr')} [list 1 $err] } + +#------------------------------------------------------------------------- +# Experiment with a tokenizer that considers " to be a token character. +# +do_execsql_test 4.0 { + SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"'); +} {{"a" AND """"}} finish_test Index: ext/fts5/test/fts5fault1.test ================================================================== --- ext/fts5/test/fts5fault1.test +++ ext/fts5/test/fts5fault1.test @@ -30,16 +30,16 @@ # 4: MATCH expressions # # faultsim_save_and_close -do_faultsim_test 1 -prep { +do_faultsim_test 1 -faults ioerr-t* -prep { faultsim_restore_and_reopen } -body { execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3') } } -test { - faultsim_test_result {0 {}} + faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} } reset_db do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix='1, 2, 3'); Index: ext/fts5/test/fts5fault4.test ================================================================== --- ext/fts5/test/fts5fault4.test +++ ext/fts5/test/fts5fault4.test @@ -199,12 +199,10 @@ db eval { SELECT previc(x3) FROM x3 WHERE x3 MATCH 'a' } } -test { faultsim_test_result {0 {0 2 7}} {1 SQLITE_NOMEM} } -} - #------------------------------------------------------------------------- # OOM error when querying for a phrase with many tokens. # reset_db do_execsql_test 7.0 { @@ -301,10 +299,40 @@ do_faultsim_test 9.1 -faults oom-* -body { db eval { SELECT rowid FROM tt WHERE tt MATCH 'a NOT x' } } -test { faultsim_test_result {0 {50 100 150 200}} {1 SQLITE_NOMEM} } + +#------------------------------------------------------------------------- +# OOM in fts5_expr() SQL function. +# +reset_db +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE tt USING fts5(x); + INSERT INTO tt(tt, rank) VALUES('pgsz', 32); + BEGIN; + WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<200) + INSERT INTO tt(rowid, x) + SELECT i, CASE WHEN (i%50)==0 THEN 'a a a a a a' ELSE 'a x a x a x' END + FROM ii; + COMMIT; +} + +} + +do_faultsim_test 10.1 -faults oom-t* -body { + db one { SELECT fts5_expr('a AND b NEAR(a b)') } +} -test { + faultsim_test_result {0 {"a" AND ("b" AND NEAR("a" "b", 10))}} +} + +#do_faultsim_test 10.2 -faults oom-t* -body { +# db one { SELECT fts5_expr_tcl('x:"a b c" AND b NEAR(a b)', 'ns', 'x') } +#} -test { +# set res {[ns -col 0 -- {a b c}] && ([ns -- {b}] && [ns -near 10 -- {a} {b}]} +# faultsim_test_result [list 0 $res] +#} finish_test ADDED ext/fts5/test/fts5integrity.test Index: ext/fts5/test/fts5integrity.test ================================================================== --- ext/fts5/test/fts5integrity.test +++ ext/fts5/test/fts5integrity.test @@ -0,0 +1,35 @@ +# 2015 Jan 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file containst tests focused on the integrity-check procedure. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5integrity + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xx USING fts5(x); + INSERT INTO xx VALUES('term'); +} +do_execsql_test 1.1 { + INSERT INTO xx(xx) VALUES('integrity-check'); +} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE yy USING fts5(x, prefix=1); + INSERT INTO yy VALUES('term'); +} +do_execsql_test 2.1 { + INSERT INTO yy(yy) VALUES('integrity-check'); +} + +finish_test + Index: ext/fts5/test/fts5prefix.test ================================================================== --- ext/fts5/test/fts5prefix.test +++ ext/fts5/test/fts5prefix.test @@ -13,49 +13,49 @@ # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5prefix +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xx USING fts5(x, prefix=1); + INSERT INTO xx VALUES('one two three'); + INSERT INTO xx VALUES('four five six'); + INSERT INTO xx VALUES('seven eight nine ten'); +} + +do_execsql_test 1.1 { + SELECT rowid FROM xx WHERE xx MATCH 't*' +} {1 3} + #------------------------------------------------------------------------- # Check that prefix indexes really do index n-character prefixes, not # n-byte prefixes. Use the ascii tokenizer so as not to be confused by # diacritic removal. # -do_execsql_test 1.0 { +do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = ascii, prefix = 2) } -do_test 1.2 { +do_test 2.1 { foreach {rowid string} { 1 "\xCA\xCB\xCC\xCD" 2 "\u1234\u5678\u4321\u8765" } { execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $string) } } } {} -do_execsql_test 1.1.2 { +do_execsql_test 2.2 { INSERT INTO t1(t1) VALUES('integrity-check'); } -#db eval { select fts5_decode(id, block) AS d FROM t1_data; } { puts $d } - -foreach o {1 2} { - if {$o==2} breakpoint - foreach {tn q res} { - 1 "SELECT rowid FROM t1 WHERE t1 MATCH '\xCA\xCB*'" 1 - 2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2 - } { - do_execsql_test 1.$o.$tn $q $res - } - - execsql { - DELETE FROM t1_data WHERE - rowid>=fts5_rowid('start-of-index', 0) AND - rowid=$nOpt } usage set O(crisismerge) [lindex $argv $i] } + + -prefix { + if { [incr i]>=$nOpt } usage + set O(prefix) [lindex $argv $i] + } default { usage } } @@ -96,12 +103,14 @@ if {$O(delete)} { file delete -force $dbfile } sqlite3 db $dbfile db func loadfile loadfile db transaction { + set pref "" + if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" } catch { - db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok))" + db eval "CREATE VIRTUAL TABLE t1 USING $O(vtab) (path, content$O(tok)$pref)" } if {$O(automerge)>=0} { if {$O(vtab) == "fts5"} { db eval { INSERT INTO t1(t1, rank) VALUES('automerge', $O(automerge)) } } else {