Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -146,11 +146,11 @@ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; }; /* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 2 +#define FTS5_CURRENT_VERSION 3 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -15,11 +15,11 @@ #include "fts5Int.h" -#define FTS5_DEFAULT_PAGE_SIZE 1000 +#define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) Index: ext/fts5/fts5_hash.c ================================================================== --- ext/fts5/fts5_hash.c +++ ext/fts5/fts5_hash.c @@ -131,20 +131,20 @@ } memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); pHash->nEntry = 0; } -static unsigned int fts5HashKey(int nSlot, const char *p, int n){ +static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ int i; unsigned int h = 13; for(i=n-1; i>=0; i--){ h = (h << 3) ^ h ^ p[i]; } return (h % nSlot); } -static unsigned int fts5HashKey2(int nSlot, char b, const char *p, int n){ +static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ int i; unsigned int h = 13; for(i=n-1; i>=0; i--){ h = (h << 3) ^ h ^ p[i]; } @@ -168,11 +168,11 @@ for(i=0; inSlot; i++){ while( apOld[i] ){ int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, p->zKey, strlen(p->zKey)); + iHash = fts5HashKey(nNew, (u8*)p->zKey, strlen(p->zKey)); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } } @@ -208,16 +208,17 @@ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ - unsigned int iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); + unsigned int iHash; Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ /* Attempt to locate an existing hash entry */ + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( p->zKey[0]==bByte && memcmp(&p->zKey[1], pToken, nToken)==0 && p->zKey[nToken+1]==0 ){ @@ -231,20 +232,20 @@ if( nByte<128 ) nByte = 128; if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; - iHash = fts5HashKey2(pHash->nSlot, bByte, pToken, nToken); + iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); - assert( iHash==fts5HashKey(pHash->nSlot, p->zKey, nToken+1) ); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); p->iSzPoslist = p->nData; p->nData += 1; @@ -412,11 +413,11 @@ Fts5Hash *pHash, /* Hash table to query */ const char *pTerm, int nTerm, /* Query term */ const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ int *pnDoclist /* OUT: Size of doclist in bytes */ ){ - unsigned int iHash = fts5HashKey(pHash->nSlot, pTerm, nTerm); + unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -285,13 +285,12 @@ ** Each time a blob is read from the %_data table, it is padded with this ** many zero bytes. This makes it easier to decode the various record formats ** without overreading if the records are corrupt. */ #define FTS5_DATA_ZERO_PADDING 8 +#define FTS5_DATA_PADDING 20 -typedef struct Fts5BtreeIter Fts5BtreeIter; -typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; typedef struct Fts5NodeIter Fts5NodeIter; @@ -331,10 +330,13 @@ /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ + sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ + sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { u8 *a; @@ -385,21 +387,24 @@ i64 iPrev; /* Previous docid value written to page */ Fts5Buffer buf; /* Buffer containing page data */ }; struct Fts5SegWriter { int iSegid; /* Segid to write to */ - int nWriter; /* Number of entries in aWriter */ - Fts5PageWriter *aWriter; /* Array of PageWriter objects */ + Fts5PageWriter writer; /* PageWriter object */ i64 iPrevRowid; /* Previous docid written to current leaf */ u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ u8 bFirstRowidInPage; /* True if next rowid is first in page */ u8 bFirstTermInPage; /* True if next term will be first in leaf */ int nLeafWritten; /* Number of leaf pages written */ int nEmpty; /* Number of contiguous term-less nodes */ int nDlidx; /* Allocated size of aDlidx[] array */ Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ + + /* Values to insert into the %_idx table */ + Fts5Buffer btterm; /* TODO: Docs */ + int iBtPage; /* TODO: This */ }; /* ** Object for iterating through the merged results of one or more segments, ** visiting each term/docid pair in the merged data. @@ -567,47 +572,10 @@ int iSegid; Fts5DlidxLvl aLvl[1]; }; - -/* -** An Fts5BtreeIter object is used to iterate through all entries in the -** b-tree hierarchy belonging to a single fts5 segment. In this case the -** "b-tree hierarchy" is all b-tree nodes except leaves. Each entry in the -** b-tree hierarchy consists of the following: -** -** iLeaf: The page number of the leaf page the entry points to. -** -** term: A split-key that all terms on leaf page $iLeaf must be greater -** than or equal to. The "term" associated with the first b-tree -** hierarchy entry (the one that points to leaf page 1) is always -** an empty string. -** -** nEmpty: The number of empty (termless) leaf pages that immediately -** following iLeaf. -** -** The Fts5BtreeIter object is only used as part of the integrity-check code. -*/ -struct Fts5BtreeIterLevel { - Fts5NodeIter s; /* Iterator for the current node */ - Fts5Data *pData; /* Data for the current node */ -}; -struct Fts5BtreeIter { - Fts5Index *p; /* FTS5 backend object */ - Fts5StructureSegment *pSeg; /* Iterate through this segment's b-tree */ - int nLvl; /* Size of aLvl[] array */ - Fts5BtreeIterLevel *aLvl; /* Level for each tier of b-tree */ - - /* Output variables */ - Fts5Buffer term; /* Current term */ - int iLeaf; /* Leaf containing terms >= current term */ - int nEmpty; /* Number of "empty" leaves following iLeaf */ - int bEof; /* Set to true at EOF */ - int bDlidx; /* True if there exists a dlidx */ -}; - /* ** The first argument passed to this macro is a pointer to an Fts5Buffer ** object. */ @@ -746,11 +714,11 @@ aOut = pBuf->p; if( aOut==0 ){ rc = SQLITE_NOMEM; } }else{ - int nSpace = nByte + FTS5_DATA_ZERO_PADDING; + int nSpace = nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data)); if( pRet ){ pRet->n = nByte; aOut = pRet->p = (u8*)&pRet[1]; }else{ @@ -802,10 +770,27 @@ ** fts5DataRead(). */ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } + +static int fts5IndexPrepareStmt( + Fts5Index *p, + sqlite3_stmt **ppStmt, + char *zSql +){ + if( p->rc==SQLITE_OK ){ + if( zSql ){ + p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); + }else{ + p->rc = SQLITE_NOMEM; + } + } + sqlite3_free(zSql); + return p->rc; +} + /* ** INSERT OR REPLACE a record into the %_data table. */ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ @@ -812,21 +797,15 @@ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; - char *zSql = sqlite3Fts5Mprintf(&rc, - "REPLACE INTO '%q'.%Q(id, block) VALUES(?,?)", pConfig->zDb, p->zDataTbl - ); - if( zSql ){ - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pWriter, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ){ - p->rc = rc; - return; - } + fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", + pConfig->zDb, pConfig->zName + )); + if( p->rc ) return; } sqlite3_bind_int64(p->pWriter, 1, iRowid); sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); sqlite3_step(p->pWriter); @@ -843,11 +822,12 @@ if( p->pDeleter==0 ){ int rc; Fts5Config *pConfig = p->pConfig; char *zSql = sqlite3_mprintf( - "DELETE FROM '%q'.%Q WHERE id>=? AND id<=?", pConfig->zDb, p->zDataTbl + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", + pConfig->zDb, pConfig->zName ); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0); @@ -870,10 +850,22 @@ */ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1; fts5DataDelete(p, iFirst, iLast); + if( p->pIdxDeleter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE segid=?", + pConfig->zDb, pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); + sqlite3_step(p->pIdxDeleter); + p->rc = sqlite3_reset(p->pIdxDeleter); + } } /* ** Release a reference to an Fts5Structure object returned by an earlier ** call to fts5StructureRead() or fts5StructureDecode(). @@ -2332,16 +2324,27 @@ memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ - for(h=pSeg->nHeight-1; h>0; h--){ - i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg); - fts5DataBuffer(p, pBuf, iRowid); - if( p->rc ) break; - iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx); + if( p->pIdxSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term<=? ORDER BY term DESC LIMIT 1", + pConfig->zDb, pConfig->zName + )); } + if( p->rc ) return; + sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); + sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ + i64 val = sqlite3_column_int(p->pIdxSelect, 0); + iPg = (val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(p->pIdxSelect); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; bDlidx = 0; } @@ -3167,56 +3170,55 @@ } return p->rc; } /* -** If an "nEmpty" record must be written to the b-tree before the next -** term, write it now. +** If the current doclist-index accumulating in pWriter->aDlidx[] is large +** enough, flush it to disk and return 1. Otherwise discard it and return +** zero. +*/ +static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag = 0; + + /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written + ** to the database, also write the doclist-index to disk. */ + if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ + bFlag = 1; + } + fts5WriteDlidxClear(p, pWriter, bFlag); + pWriter->nEmpty = 0; + return bFlag; +} + +/* +** This function is called whenever processing of the doclist for the +** last term on leaf page (pWriter->iBtPage) is completed. +** +** The doclist-index for that term is currently stored in-memory within the +** Fts5SegWriter.aDlidx[] array. If it is large enough, this function +** writes it out to disk. Or, if it is too small to bother with, discards +** it. +** +** Fts5SegWriter.btterm currently contains the first term on page iBtPage. */ -static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){ - if( pWriter->nEmpty ){ - int bFlag = 0; - Fts5PageWriter *pPg; - pPg = &pWriter->aWriter[1]; - - /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written - ** to the database, also write the doclist-index to disk. */ - if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ - bFlag = 1; - } - fts5WriteDlidxClear(p, pWriter, bFlag); - fts5BufferAppendVarint(&p->rc, &pPg->buf, bFlag); - fts5BufferAppendVarint(&p->rc, &pPg->buf, pWriter->nEmpty); - pWriter->nEmpty = 0; - }else{ - fts5WriteDlidxClear(p, pWriter, 0); - } - - assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].buf.n==0 ); - assert( pWriter->nDlidx==0 || pWriter->aDlidx[0].bPrevValid==0 ); -} - -static void fts5WriteBtreeGrow(Fts5Index *p, Fts5SegWriter *pWriter){ +static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ + int bFlag; + + assert( pWriter->iBtPage || pWriter->nEmpty==0 ); + if( pWriter->iBtPage==0 ) return; + bFlag = fts5WriteFlushDlidx(p, pWriter); + if( p->rc==SQLITE_OK ){ - Fts5PageWriter *aNew; - Fts5PageWriter *pNew; - int nNew = sizeof(Fts5PageWriter) * (pWriter->nWriter+1); - - aNew = (Fts5PageWriter*)sqlite3_realloc(pWriter->aWriter, nNew); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - - pNew = &aNew[pWriter->nWriter]; - memset(pNew, 0, sizeof(Fts5PageWriter)); - pNew->pgno = 1; - fts5BufferAppendVarint(&p->rc, &pNew->buf, 1); - - pWriter->nWriter++; - pWriter->aWriter = aNew; - } + const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); + /* The following was already done in fts5WriteInit(): */ + /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ + sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); + sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); + sqlite3_step(p->pIdxWriter); + p->rc = sqlite3_reset(p->pIdxWriter); + } + pWriter->iBtPage = 0; } /* ** This is called once for each leaf page except the first that contains ** at least one term. Argument (nTerm/pTerm) is the split-key - a term that @@ -3229,40 +3231,13 @@ static void fts5WriteBtreeTerm( Fts5Index *p, /* FTS5 backend object */ Fts5SegWriter *pWriter, /* Writer object */ int nTerm, const u8 *pTerm /* First term on new page */ ){ - int iHeight; - for(iHeight=1; 1; iHeight++){ - Fts5PageWriter *pPage; - - if( iHeight>=pWriter->nWriter ){ - fts5WriteBtreeGrow(p, pWriter); - if( p->rc ) return; - } - pPage = &pWriter->aWriter[iHeight]; - - fts5WriteBtreeNEmpty(p, pWriter); - - if( pPage->buf.n>=p->pConfig->pgsz ){ - /* pPage will be written to disk. The term will be written into the - ** parent of pPage. */ - i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, iHeight, pPage->pgno); - fts5DataWrite(p, iRowid, pPage->buf.p, pPage->buf.n); - fts5BufferZero(&pPage->buf); - fts5BufferZero(&pPage->term); - fts5BufferAppendVarint(&p->rc, &pPage->buf, pPage[-1].pgno); - pPage->pgno++; - }else{ - int nPre = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); - fts5BufferAppendVarint(&p->rc, &pPage->buf, nPre+2); - fts5BufferAppendVarint(&p->rc, &pPage->buf, nTerm-nPre); - fts5BufferAppendBlob(&p->rc, &pPage->buf, nTerm-nPre, pTerm+nPre); - fts5BufferSet(&p->rc, &pPage->term, nTerm, pTerm); - break; - } - } + fts5WriteFlushBtree(p, pWriter); + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); + pWriter->iBtPage = pWriter->writer.pgno; } /* ** This function is called when flushing a leaf page that contains no ** terms at all to disk. @@ -3342,11 +3317,11 @@ } if( pDlidx->bPrevValid ){ iVal = iRowid - pDlidx->iPrev; }else{ - i64 iPgno = (i==0 ? pWriter->aWriter[0].pgno : pDlidx[-1].pgno); + i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); iVal = iRowid; } @@ -3357,11 +3332,11 @@ } } static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; if( pWriter->bFirstTermInPage ){ /* No term was written to this page. */ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); @@ -3396,11 +3371,11 @@ Fts5Index *p, Fts5SegWriter *pWriter, int nTerm, const u8 *pTerm ){ int nPrefix; /* Bytes of prefix compression for term */ - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; assert( pPage->buf.n==0 || pPage->buf.n>4 ); if( pPage->buf.n==0 ){ /* Zero the first term and first docid fields */ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; @@ -3431,11 +3406,11 @@ int n = nTerm; if( pPage->term.n ){ n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); - pPage = &pWriter->aWriter[0]; + pPage = &pWriter->writer; } }else{ nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); } @@ -3469,11 +3444,11 @@ Fts5SegWriter *pWriter, i64 iRowid, int nPos ){ if( p->rc==SQLITE_OK ){ - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; /* If this is to be the first docid written to the page, set the ** docid-pointer in the page-header. Also append a value to the dlidx ** buffer, in case a doclist-index is required. */ if( pWriter->bFirstRowidInPage ){ @@ -3504,11 +3479,11 @@ Fts5Index *p, Fts5SegWriter *pWriter, const u8 *aData, int nData ){ - Fts5PageWriter *pPage = &pWriter->aWriter[0]; + Fts5PageWriter *pPage = &pWriter->writer; const u8 *a = aData; int n = nData; assert( p->pConfig->pgsz>0 ); while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){ @@ -3527,11 +3502,11 @@ fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); } } static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ - fts5BufferAppendVarint(&p->rc, &pWriter->aWriter[0].buf, 0); + fts5BufferAppendVarint(&p->rc, &pWriter->writer.buf, 0); } /* ** Flush any data cached by the writer object to the database. Free any ** allocations associated with the writer. @@ -3541,43 +3516,28 @@ Fts5SegWriter *pWriter, /* Writer object */ int *pnHeight, /* OUT: Height of the b-tree */ int *pnLeaf /* OUT: Number of leaf pages in b-tree */ ){ int i; + Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ - Fts5PageWriter *pLeaf = &pWriter->aWriter[0]; if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){ *pnLeaf = 0; *pnHeight = 0; }else{ if( pLeaf->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; - if( pWriter->nWriter==1 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ - fts5WriteBtreeGrow(p, pWriter); - } - if( pWriter->nWriter>1 ){ - fts5WriteBtreeNEmpty(p, pWriter); - } - *pnHeight = pWriter->nWriter; - - for(i=1; inWriter; i++){ - Fts5PageWriter *pPg = &pWriter->aWriter[i]; - fts5DataWrite(p, - FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pPg->pgno), - pPg->buf.p, pPg->buf.n - ); - } - } - } - for(i=0; inWriter; i++){ - Fts5PageWriter *pPg = &pWriter->aWriter[i]; - fts5BufferFree(&pPg->term); - fts5BufferFree(&pPg->buf); - } - sqlite3_free(pWriter->aWriter); + + fts5WriteFlushBtree(p, pWriter); + *pnHeight = 0; + } + } + fts5BufferFree(&pLeaf->term); + fts5BufferFree(&pLeaf->buf); + fts5BufferFree(&pWriter->btterm); for(i=0; inDlidx; i++){ sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); } sqlite3_free(pWriter->aDlidx); @@ -3589,52 +3549,25 @@ int iSegid ){ memset(pWriter, 0, sizeof(Fts5SegWriter)); pWriter->iSegid = iSegid; - pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, sizeof(Fts5PageWriter)); - if( fts5WriteDlidxGrow(p, pWriter, 1) ) return; - pWriter->nWriter = 1; - pWriter->nDlidx = 1; - pWriter->aWriter[0].pgno = 1; - pWriter->bFirstTermInPage = 1; -} - -static void fts5WriteInitForAppend( - Fts5Index *p, /* FTS5 backend object */ - Fts5SegWriter *pWriter, /* Writer to initialize */ - Fts5StructureSegment *pSeg /* Segment object to append to */ -){ - int nByte = pSeg->nHeight * sizeof(Fts5PageWriter); - memset(pWriter, 0, sizeof(Fts5SegWriter)); - pWriter->iSegid = pSeg->iSegid; - pWriter->aWriter = (Fts5PageWriter*)fts5IdxMalloc(p, nByte); - pWriter->aDlidx = (Fts5DlidxWriter*)fts5IdxMalloc(p, sizeof(Fts5DlidxWriter)); - - if( p->rc==SQLITE_OK ){ - int pgno = 1; - int i; - pWriter->nDlidx = 1; - pWriter->nWriter = pSeg->nHeight; - pWriter->aWriter[0].pgno = pSeg->pgnoLast+1; - for(i=pSeg->nHeight-1; i>0; i--){ - i64 iRowid = FTS5_SEGMENT_ROWID(pWriter->iSegid, i, pgno); - Fts5PageWriter *pPg = &pWriter->aWriter[i]; - pPg->pgno = pgno; - fts5DataBuffer(p, &pPg->buf, iRowid); - if( p->rc==SQLITE_OK ){ - Fts5NodeIter ss; - fts5NodeIterInit(pPg->buf.p, pPg->buf.n, &ss); - while( ss.aData ) fts5NodeIterNext(&p->rc, &ss); - fts5BufferSet(&p->rc, &pPg->term, ss.term.n, ss.term.p); - pgno = ss.iChild; - fts5NodeIterFree(&ss); - } - } - assert( p->rc!=SQLITE_OK || (pgno+pWriter->nEmpty)==pSeg->pgnoLast ); - pWriter->bFirstTermInPage = 1; - assert( pWriter->aWriter[0].term.n==0 ); + fts5WriteDlidxGrow(p, pWriter, 1); + pWriter->writer.pgno = 1; + pWriter->bFirstTermInPage = 1; + pWriter->iBtPage = 1; + + if( p->pIdxWriter==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", + pConfig->zDb, pConfig->zName + )); + } + + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); } } /* ** Iterator pIter was used to iterate through the input segments of on an @@ -3670,11 +3603,11 @@ fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; - fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1),iLeafRowid); + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid); fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } } fts5BufferFree(&buf); @@ -3717,12 +3650,15 @@ memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ pLvlOut = &pStruct->aLevel[iLvl+1]; assert( pLvlOut->nSeg>0 ); nInput = pLvl->nMerge; - fts5WriteInitForAppend(p, &writer, &pLvlOut->aSeg[pLvlOut->nSeg-1]); pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; + + fts5WriteInit(p, &writer, pSeg->iSegid); + writer.writer.pgno = pSeg->pgnoLast+1; + writer.iBtPage = 0; }else{ int iSegid = fts5AllocateSegid(p, pStruct); /* Extend the Fts5Structure object as required to ensure the output ** segment exists. */ @@ -3809,11 +3745,11 @@ if( pSeg->pgnoLast==0 ){ pLvlOut->nSeg--; pStruct->nSegment--; } }else{ - assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 ); + assert( pSeg->pgnoLast>0 ); fts5TrimSegments(p, pIter); pLvl->nMerge = nInput; } fts5MultiIterFree(p, pIter); @@ -3984,11 +3920,11 @@ fts5WriteInit(p, &writer, iSegid); /* Pre-allocate the buffer used to assemble leaf pages to the target ** page size. */ assert( pgsz>0 ); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; fts5BufferGrow(&p->rc, pBuf, pgsz + 20); /* Begin scanning through hash table entries. This loop runs once for each ** term/doclist currently stored within the hash table. */ if( p->rc==SQLITE_OK ){ @@ -4008,11 +3944,11 @@ /* Decide if the term will fit on the current leaf. If it will not, ** flush the leaf to disk here. */ if( (pBuf->n + nTerm + 2) > pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; if( (nTerm + 32) > pBuf->nSpace ){ fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); if( p->rc ) break; } } @@ -4025,14 +3961,14 @@ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nPre); nSuffix = nTerm - nPre; }else{ fts5PutU16(&pBuf->p[2], pBuf->n); writer.bFirstTermInPage = 0; - if( writer.aWriter[0].pgno!=1 ){ + if( writer.writer.pgno!=1 ){ int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; assert( nPren += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nSuffix); @@ -4040,11 +3976,11 @@ /* We just wrote a term into page writer.aWriter[0].pgno. If a ** doclist-index is to be generated for this doclist, it will be ** associated with this page. */ assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 ); - writer.aDlidx[0].pgno = writer.aWriter[0].pgno; + writer.aDlidx[0].pgno = writer.writer.pgno; if( pgsz>=(pBuf->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ @@ -4097,11 +4033,11 @@ assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; if( pBuf->n>=pgsz ){ fts5WriteFlushLeaf(p, &writer); - pBuf = &writer.aWriter[0].buf; + pBuf = &writer.writer.buf; } if( iPos>=nCopy ) break; } } iOff += nCopy; @@ -4130,11 +4066,10 @@ pSeg->pgnoLast = pgnoLast; pStruct->nSegment++; } fts5StructurePromote(p, 0, pStruct); } - fts5IndexAutomerge(p, &pStruct, pgnoLast); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); @@ -4559,10 +4494,16 @@ p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); if( p->zDataTbl && bCreate ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5CreateTable(pConfig, "idx", + "segid, term, pgno, PRIMARY KEY(segid, term)", + 1, pzErr + ); + } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p); } } } @@ -4582,10 +4523,13 @@ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); + sqlite3_finalize(p->pIdxWriter); + sqlite3_finalize(p->pIdxDeleter); + sqlite3_finalize(p->pIdxSelect); sqlite3Fts5HashFree(p->pHash); sqlite3Fts5BufferFree(&p->scratch); sqlite3_free(p->zDataTbl); sqlite3_free(p); } @@ -4931,96 +4875,10 @@ if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); for(i=0; iaLvl[0]) * (pSeg->nHeight-1); - memset(pIter, 0, sizeof(*pIter)); - if( nByte ){ - pIter->aLvl = (Fts5BtreeIterLevel*)fts5IdxMalloc(p, nByte); - } - if( p->rc==SQLITE_OK ){ - pIter->nLvl = pSeg->nHeight-1; - pIter->p = p; - pIter->pSeg = pSeg; - } - for(i=0; p->rc==SQLITE_OK && inLvl; i++){ - i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, i+1, 1); - Fts5Data *pData; - pIter->aLvl[i].pData = pData = fts5DataRead(p, iRowid); - if( pData ){ - fts5NodeIterInit(pData->p, pData->n, &pIter->aLvl[i].s); - } - } - - if( pIter->nLvl==0 || p->rc ){ - pIter->bEof = 1; - pIter->iLeaf = pSeg->pgnoLast; - }else{ - pIter->nEmpty = pIter->aLvl[0].s.nEmpty; - pIter->iLeaf = pIter->aLvl[0].s.iChild; - pIter->bDlidx = pIter->aLvl[0].s.bDlidx; - } -} - -static void fts5BtreeIterNext(Fts5BtreeIter *pIter){ - Fts5Index *p = pIter->p; - int i; - - assert( pIter->bEof==0 && pIter->aLvl[0].s.aData ); - for(i=0; inLvl && p->rc==SQLITE_OK; i++){ - Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - fts5NodeIterNext(&p->rc, &pLvl->s); - if( pLvl->s.aData ){ - fts5BufferSet(&p->rc, &pIter->term, pLvl->s.term.n, pLvl->s.term.p); - break; - }else{ - fts5NodeIterFree(&pLvl->s); - fts5DataRelease(pLvl->pData); - pLvl->pData = 0; - } - } - if( i==pIter->nLvl || p->rc ){ - pIter->bEof = 1; - }else{ - int iSegid = pIter->pSeg->iSegid; - for(i--; i>=0; i--){ - Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - i64 iRowid = FTS5_SEGMENT_ROWID(iSegid, i+1, pLvl[1].s.iChild); - pLvl->pData = fts5DataRead(p, iRowid); - if( pLvl->pData ){ - fts5NodeIterInit(pLvl->pData->p, pLvl->pData->n, &pLvl->s); - } - } - } - - pIter->nEmpty = pIter->aLvl[0].s.nEmpty; - pIter->bDlidx = pIter->aLvl[0].s.bDlidx; - pIter->iLeaf = pIter->aLvl[0].s.iChild; -} - -static void fts5BtreeIterFree(Fts5BtreeIter *pIter){ - int i; - for(i=0; inLvl; i++){ - Fts5BtreeIterLevel *pLvl = &pIter->aLvl[i]; - fts5NodeIterFree(&pLvl->s); - if( pLvl->pData ){ - fts5DataRelease(pLvl->pData); - pLvl->pData = 0; - } - } - sqlite3_free(pIter->aLvl); - fts5BufferFree(&pIter->term); -} - #ifdef SQLITE_DEBUG /* ** This function is purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. ** @@ -5163,38 +5021,79 @@ #else # define fts5TestDlidxReverse(x,y,z) # define fts5TestTerm(u,v,w,x,y,z) #endif + +/* +** Check that: +** +** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and +** contain zero terms. +** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and +** contain zero rowids. +*/ +static void fts5IndexIntegrityCheckEmpty( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ + int iFirst, + int iNoRowid, + int iLast +){ + int i; + + /* Now check that the iter.nEmpty leaves following the current leaf + ** (a) exist and (b) contain no terms. */ + for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ + Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); + if( pLeaf ){ + if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT; + if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT; + } + fts5DataRelease(pLeaf); + if( p->rc ) break; + } +} static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ - Fts5BtreeIter iter; /* Used to iterate through b-tree hierarchy */ + Fts5Config *pConfig = p->pConfig; + sqlite3_stmt *pStmt = 0; + int rc2; + int iIdxPrevLeaf = pSeg->pgnoFirst-1; + int iDlidxPrevLeaf = pSeg->pgnoLast; if( pSeg->pgnoFirst==0 ) return; + + fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( + "SELECT segid, term, (pgno>>1), (pgno & 1) FROM '%q'.'%q_idx' WHERE segid=%d", + pConfig->zDb, pConfig->zName, pSeg->iSegid + )); /* Iterate through the b-tree hierarchy. */ - for(fts5BtreeIterInit(p, pSeg, &iter); - p->rc==SQLITE_OK && iter.bEof==0; - fts5BtreeIterNext(&iter) - ){ + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ int iOff; /* Offset of first term on leaf */ int i; /* Used to iterate through empty leaves */ + + int nIdxTerm = sqlite3_column_bytes(pStmt, 1); + const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); + int iIdxLeaf = sqlite3_column_int(pStmt, 2); + int bIdxDlidx = sqlite3_column_int(pStmt, 3); /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ - if( iter.iLeafpgnoFirst ) continue; - iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iter.iLeaf); + if( iIdxLeafpgnoFirst ) continue; + iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf); pLeaf = fts5DataRead(p, iRow); if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal - ** to or larger than the split-key in iter.term. Also check that if there + ** to or larger than the split-key in zIdxTerm. Also check that if there ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ iOff = fts5GetU16(&pLeaf->p[2]); if( iOff==0 ){ p->rc = FTS5_CORRUPT; @@ -5206,38 +5105,35 @@ iRowidOff = fts5GetU16(&pLeaf->p[0]); if( iRowidOff>=iOff ){ p->rc = FTS5_CORRUPT; }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); - res = memcmp(&pLeaf->p[iOff], iter.term.p, MIN(nTerm, iter.term.n)); - if( res==0 ) res = nTerm - iter.term.n; + res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); + if( res==0 ) res = nTerm - nIdxTerm; if( res<0 ) p->rc = FTS5_CORRUPT; } } fts5DataRelease(pLeaf); if( p->rc ) break; /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ - for(i=1; p->rc==SQLITE_OK && i<=iter.nEmpty; i++){ - pLeaf = fts5DataRead(p, iRow+i); - if( pLeaf && 0!=fts5GetU16(&pLeaf->p[2]) ){ - p->rc = FTS5_CORRUPT; - } - fts5DataRelease(pLeaf); - } + fts5IndexIntegrityCheckEmpty( + p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 + ); + if( p->rc ) break; /* If there is a doclist-index, check that it looks right. */ - if( iter.bDlidx ){ + if( bIdxDlidx ){ Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ - int iPrevLeaf = iter.iLeaf; + int iPrevLeaf = iIdxLeaf; int iSegid = pSeg->iSegid; int iPg; i64 iKey; - for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iter.iLeaf); + for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); fts5DlidxIterEof(p, pDlidx)==0; fts5DlidxIterNext(p, pDlidx) ){ /* Check any rowid-less pages that occur before the current leaf. */ @@ -5266,30 +5162,30 @@ } fts5DataRelease(pLeaf); } } - for(iPg=iPrevLeaf+1; iPg<=(iter.iLeaf + iter.nEmpty); iPg++){ - iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); - pLeaf = fts5DataRead(p, iKey); - if( pLeaf ){ - if( fts5GetU16(&pLeaf->p[0])!=0 ) p->rc = FTS5_CORRUPT; - fts5DataRelease(pLeaf); - } - } - + iDlidxPrevLeaf = iPg; fts5DlidxIterFree(pDlidx); - fts5TestDlidxReverse(p, iSegid, iter.iLeaf); + fts5TestDlidxReverse(p, iSegid, iIdxLeaf); + }else{ + iDlidxPrevLeaf = pSeg->pgnoLast; + /* TODO: Check there is no doclist index */ } + + iIdxPrevLeaf = iIdxLeaf; } + rc2 = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ +#if 0 if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ p->rc = FTS5_CORRUPT; } - - fts5BtreeIterFree(&iter); +#endif } /* ** Run internal checks to ensure that the FTS index (a) is internally Index: ext/fts5/fts5_storage.c ================================================================== --- ext/fts5/fts5_storage.c +++ ext/fts5/fts5_storage.c @@ -178,11 +178,13 @@ ** code otherwise. */ int sqlite3Fts5DropAll(Fts5Config *pConfig){ int rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_data';" + "DROP TABLE IF EXISTS %Q.'%q_idx';" "DROP TABLE IF EXISTS %Q.'%q_config';", + pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, @@ -216,10 +218,11 @@ int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ Fts5Config *pConfig = pStorage->pConfig; int rc = sqlite3Fts5StorageSync(pStorage, 1); fts5StorageRenameOne(pConfig, &rc, "data", zName); + fts5StorageRenameOne(pConfig, &rc, "idx", zName); fts5StorageRenameOne(pConfig, &rc, "config", zName); if( pConfig->bColumnsize ){ fts5StorageRenameOne(pConfig, &rc, "docsize", zName); } if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ @@ -599,11 +602,13 @@ Fts5Config *pConfig = p->pConfig; int rc; /* Delete the contents of the %_data and %_docsize tables. */ rc = fts5ExecPrintf(pConfig->db, 0, - "DELETE FROM %Q.'%q_data';", + "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_idx';", + pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, "DELETE FROM %Q.'%q_docsize';", Index: ext/fts5/test/fts5aa.test ================================================================== --- ext/fts5/test/fts5aa.test +++ ext/fts5/test/fts5aa.test @@ -25,10 +25,11 @@ CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; } { t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} + t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} } @@ -45,13 +46,14 @@ CREATE VIRTUAL TABLE t1 USING fts5(x,y); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); } + do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } -} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=1 leaves=1..1}}}/} +} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} foreach w {a b c d e f} { do_execsql_test 2.3.$w.asc { SELECT rowid FROM t1 WHERE t1 MATCH $w; } {1} Index: ext/fts5/test/fts5content.test ================================================================== --- ext/fts5/test/fts5content.test +++ ext/fts5/test/fts5content.test @@ -245,11 +245,11 @@ # reset_db do_execsql_test 6.1 { CREATE VIRTUAL TABLE xx USING fts5(x, y, content=""); SELECT name FROM sqlite_master; -} {xx xx_data xx_docsize xx_config} +} {xx xx_data xx_idx xx_docsize xx_config} do_execsql_test 6.2 { DROP TABLE xx; SELECT name FROM sqlite_master; } {}