Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts5-btree-index Excluding Merge-Ins
This is equivalent to a diff from ee348b1275 to 92c9178933
2015-07-27
| ||
12:19 | Merge the fts5-btree-index branch with this one. FTS5 changes only. (check-in: 9f1dcdc3e1 user: dan tags: trunk) | |
11:49 | Bump the "version" field in the fts5 config table. (Closed-Leaf check-in: 92c9178933 user: dan tags: fts5-btree-index) | |
11:43 | Minor tweak to the SQL schema used by fts5. (check-in: 427bf09ba4 user: dan tags: fts5-btree-index) | |
10:46 | Merge latest trunk changes with this branch. (check-in: 5ec933c257 user: dan tags: fts5-btree-index) | |
2015-07-25
| ||
19:26 | Remove a duplicated "if" condition from fts3.c. (check-in: ee348b1275 user: dan tags: trunk) | |
14:26 | Fix the rowallock.test script so that it works with builds that do not support mmap mode. (check-in: 7efed5786b user: dan tags: trunk) | |
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
144 145 146 147 148 149 150 | char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; }; /* Current expected value of %_config table 'version' field */ | | | 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | char *zRankArgs; /* Arguments to rank function */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; }; /* Current expected value of %_config table 'version' field */ #define FTS5_CURRENT_VERSION 3 #define FTS5_CONTENT_NORMAL 0 #define FTS5_CONTENT_NONE 1 #define FTS5_CONTENT_EXTERNAL 2 |
︙ | ︙ |
Changes to ext/fts5/fts5_config.c.
︙ | ︙ | |||
13 14 15 16 17 18 19 | ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" | | | 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | ** This is an SQLite module implementing full-text search. */ #include "fts5Int.h" #define FTS5_DEFAULT_PAGE_SIZE 4050 #define FTS5_DEFAULT_AUTOMERGE 4 #define FTS5_DEFAULT_CRISISMERGE 16 /* Maximum allowed page size */ #define FTS5_MAX_PAGE_SIZE (128*1024) static int fts5_iswhitespace(char x){ |
︙ | ︙ |
Changes to ext/fts5/fts5_hash.c.
︙ | ︙ | |||
129 130 131 132 133 134 135 | sqlite3_free(pSlot); } } memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); pHash->nEntry = 0; } | | | | 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 | sqlite3_free(pSlot); } } memset(pHash->aSlot, 0, pHash->nSlot * sizeof(Fts5HashEntry*)); pHash->nEntry = 0; } static unsigned int fts5HashKey(int nSlot, const u8 *p, int n){ int i; unsigned int h = 13; for(i=n-1; i>=0; i--){ h = (h << 3) ^ h ^ p[i]; } return (h % nSlot); } static unsigned int fts5HashKey2(int nSlot, u8 b, const u8 *p, int n){ int i; unsigned int h = 13; for(i=n-1; i>=0; i--){ h = (h << 3) ^ h ^ p[i]; } h = (h << 3) ^ h ^ b; return (h % nSlot); |
︙ | ︙ | |||
166 167 168 169 170 171 172 | memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); for(i=0; i<pHash->nSlot; i++){ while( apOld[i] ){ int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; | | | 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 | memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); for(i=0; i<pHash->nSlot; i++){ while( apOld[i] ){ int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; iHash = fts5HashKey(nNew, (u8*)p->zKey, strlen(p->zKey)); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } } sqlite3_free(apOld); pHash->nSlot = nNew; |
︙ | ︙ | |||
206 207 208 209 210 211 212 | Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ | | > | | | 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 | Fts5Hash *pHash, i64 iRowid, /* Rowid for this entry */ int iCol, /* Column token appears in (-ve -> delete) */ int iPos, /* Position of token within column */ char bByte, /* First byte of token */ const char *pToken, int nToken /* Token to add or remove to or from index */ ){ unsigned int iHash; Fts5HashEntry *p; u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( p->zKey[0]==bByte && memcmp(&p->zKey[1], pToken, nToken)==0 && p->zKey[nToken+1]==0 ){ break; } } /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; if( (pHash->nEntry*2)>=pHash->nSlot ){ int rc = fts5HashResize(pHash); if( rc!=SQLITE_OK ) return rc; iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); } p = (Fts5HashEntry*)sqlite3_malloc(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, FTS5_HASHENTRYSIZE); p->nAlloc = nByte; p->zKey[0] = bByte; memcpy(&p->zKey[1], pToken, nToken); assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); p->zKey[nToken+1] = '\0'; p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid); p->iSzPoslist = p->nData; p->nData += 1; p->iRowid = iRowid; p->pHashNext = pHash->aSlot[iHash]; |
︙ | ︙ | |||
410 411 412 413 414 415 416 | */ int sqlite3Fts5HashQuery( Fts5Hash *pHash, /* Hash table to query */ const char *pTerm, int nTerm, /* Query term */ const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ int *pnDoclist /* OUT: Size of doclist in bytes */ ){ | | | 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 | */ int sqlite3Fts5HashQuery( Fts5Hash *pHash, /* Hash table to query */ const char *pTerm, int nTerm, /* Query term */ const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ int *pnDoclist /* OUT: Size of doclist in bytes */ ){ unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; } if( p ){ |
︙ | ︙ |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
283 284 285 286 287 288 289 290 | /* ** Each time a blob is read from the %_data table, it is padded with this ** many zero bytes. This makes it easier to decode the various record formats ** without overreading if the records are corrupt. */ #define FTS5_DATA_ZERO_PADDING 8 | > < < | 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 | /* ** Each time a blob is read from the %_data table, it is padded with this ** many zero bytes. This makes it easier to decode the various record formats ** without overreading if the records are corrupt. */ #define FTS5_DATA_ZERO_PADDING 8 #define FTS5_DATA_PADDING 20 typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5DlidxLvl Fts5DlidxLvl; typedef struct Fts5DlidxWriter Fts5DlidxWriter; typedef struct Fts5NodeIter Fts5NodeIter; typedef struct Fts5PageWriter Fts5PageWriter; typedef struct Fts5SegIter Fts5SegIter; |
︙ | ︙ | |||
329 330 331 332 333 334 335 336 337 338 339 340 341 342 | /* Error state. */ int rc; /* Current error code */ /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { u8 *a; int n; int i; | > > > | 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 | /* Error state. */ int rc; /* Current error code */ /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { u8 *a; int n; int i; |
︙ | ︙ | |||
383 384 385 386 387 388 389 | int pgno; /* Page number for this page */ int bPrevValid; /* True if iPrev is valid */ i64 iPrev; /* Previous docid value written to page */ Fts5Buffer buf; /* Buffer containing page data */ }; struct Fts5SegWriter { int iSegid; /* Segid to write to */ | < | > > > > | 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 | int pgno; /* Page number for this page */ int bPrevValid; /* True if iPrev is valid */ i64 iPrev; /* Previous docid value written to page */ Fts5Buffer buf; /* Buffer containing page data */ }; struct Fts5SegWriter { int iSegid; /* Segid to write to */ Fts5PageWriter writer; /* PageWriter object */ i64 iPrevRowid; /* Previous docid written to current leaf */ u8 bFirstRowidInDoclist; /* True if next rowid is first in doclist */ u8 bFirstRowidInPage; /* True if next rowid is first in page */ u8 bFirstTermInPage; /* True if next term will be first in leaf */ int nLeafWritten; /* Number of leaf pages written */ int nEmpty; /* Number of contiguous term-less nodes */ int nDlidx; /* Allocated size of aDlidx[] array */ Fts5DlidxWriter *aDlidx; /* Array of Fts5DlidxWriter objects */ /* Values to insert into the %_idx table */ Fts5Buffer btterm; /* TODO: Docs */ int iBtPage; /* TODO: This */ }; /* ** Object for iterating through the merged results of one or more segments, ** visiting each term/docid pair in the merged data. ** ** nSeg is always a power of two greater than or equal to the number of |
︙ | ︙ | |||
565 566 567 568 569 570 571 | struct Fts5DlidxIter { int nLvl; int iSegid; Fts5DlidxLvl aLvl[1]; }; | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 570 571 572 573 574 575 576 577 578 579 580 581 582 583 | struct Fts5DlidxIter { int nLvl; int iSegid; Fts5DlidxLvl aLvl[1]; }; /* ** The first argument passed to this macro is a pointer to an Fts5Buffer ** object. */ #define fts5BufferSize(pBuf,n) { \ if( pBuf->nSpace<n ) { \ |
︙ | ︙ | |||
744 745 746 747 748 749 750 | fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20); pBuf->n = nByte; aOut = pBuf->p; if( aOut==0 ){ rc = SQLITE_NOMEM; } }else{ | | | 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 | fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20); pBuf->n = nByte; aOut = pBuf->p; if( aOut==0 ){ rc = SQLITE_NOMEM; } }else{ int nSpace = nByte + FTS5_DATA_PADDING; pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data)); if( pRet ){ pRet->n = nByte; aOut = pRet->p = (u8*)&pRet[1]; }else{ rc = SQLITE_NOMEM; } |
︙ | ︙ | |||
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 | /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). */ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } /* ** INSERT OR REPLACE a record into the %_data table. */ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; | > > > > > > > > > > > > > > > > > | | > | < < < < < < | < | > | 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 | /* ** Release a reference to data record returned by an earlier call to ** fts5DataRead(). */ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } static int fts5IndexPrepareStmt( Fts5Index *p, sqlite3_stmt **ppStmt, char *zSql ){ if( p->rc==SQLITE_OK ){ if( zSql ){ p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); }else{ p->rc = SQLITE_NOMEM; } } sqlite3_free(zSql); return p->rc; } /* ** INSERT OR REPLACE a record into the %_data table. */ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->rc!=SQLITE_OK ) return; if( p->pWriter==0 ){ int rc = SQLITE_OK; Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", pConfig->zDb, pConfig->zName )); if( p->rc ) return; } sqlite3_bind_int64(p->pWriter, 1, iRowid); sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); sqlite3_step(p->pWriter); p->rc = sqlite3_reset(p->pWriter); } /* ** Execute the following SQL: ** ** DELETE FROM %_data WHERE id BETWEEN $iFirst AND $iLast */ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ if( p->rc!=SQLITE_OK ) return; if( p->pDeleter==0 ){ int rc; Fts5Config *pConfig = p->pConfig; char *zSql = sqlite3_mprintf( "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", pConfig->zDb, pConfig->zName ); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0); sqlite3_free(zSql); } |
︙ | ︙ | |||
868 869 870 871 872 873 874 875 876 877 878 879 880 881 | /* ** Remove all records associated with segment iSegid. */ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1; fts5DataDelete(p, iFirst, iLast); } /* ** Release a reference to an Fts5Structure object returned by an earlier ** call to fts5StructureRead() or fts5StructureDecode(). */ static void fts5StructureRelease(Fts5Structure *pStruct){ | > > > > > > > > > > > > | 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 | /* ** Remove all records associated with segment iSegid. */ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0, 0)-1; fts5DataDelete(p, iFirst, iLast); if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( "DELETE FROM '%q'.'%q_idx' WHERE segid=?", pConfig->zDb, pConfig->zName )); } if( p->rc==SQLITE_OK ){ sqlite3_bind_int(p->pIdxDeleter, 1, iSegid); sqlite3_step(p->pIdxDeleter); p->rc = sqlite3_reset(p->pIdxDeleter); } } /* ** Release a reference to an Fts5Structure object returned by an earlier ** call to fts5StructureRead() or fts5StructureDecode(). */ static void fts5StructureRelease(Fts5Structure *pStruct){ |
︙ | ︙ | |||
2330 2331 2332 2333 2334 2335 2336 | assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ | | | > > > | > > | > | > > > > > | 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 | assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; /* This block sets stack variable iPg to the leaf page number that may ** contain term (pTerm/nTerm), if it is present in the segment. */ if( p->pIdxSelect==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxSelect, sqlite3_mprintf( "SELECT pgno FROM '%q'.'%q_idx' WHERE " "segid=? AND term<=? ORDER BY term DESC LIMIT 1", pConfig->zDb, pConfig->zName )); } if( p->rc ) return; sqlite3_bind_int(p->pIdxSelect, 1, pSeg->iSegid); sqlite3_bind_blob(p->pIdxSelect, 2, pTerm, nTerm, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(p->pIdxSelect) ){ i64 val = sqlite3_column_int(p->pIdxSelect, 0); iPg = (val>>1); bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(p->pIdxSelect); if( iPg<pSeg->pgnoFirst ){ iPg = pSeg->pgnoFirst; bDlidx = 0; } pIter->iLeafPgno = iPg - 1; |
︙ | ︙ | |||
3165 3166 3167 3168 3169 3170 3171 | pWriter->nDlidx = nLvl; } } return p->rc; } /* | | > | | < | < < | | | | | | < < | | < | > > | > > | > > > > > > > | > > > < < < | | | < | < < | | | < < | | < < < < < < < | < < < | < | < < < < < < < < < < < < < < < < < < | 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 | pWriter->nDlidx = nLvl; } } return p->rc; } /* ** If the current doclist-index accumulating in pWriter->aDlidx[] is large ** enough, flush it to disk and return 1. Otherwise discard it and return ** zero. */ static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ int bFlag = 0; /* If there were FTS5_MIN_DLIDX_SIZE or more empty leaf pages written ** to the database, also write the doclist-index to disk. */ if( pWriter->aDlidx[0].buf.n>0 && pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){ bFlag = 1; } fts5WriteDlidxClear(p, pWriter, bFlag); pWriter->nEmpty = 0; return bFlag; } /* ** This function is called whenever processing of the doclist for the ** last term on leaf page (pWriter->iBtPage) is completed. ** ** The doclist-index for that term is currently stored in-memory within the ** Fts5SegWriter.aDlidx[] array. If it is large enough, this function ** writes it out to disk. Or, if it is too small to bother with, discards ** it. ** ** Fts5SegWriter.btterm currently contains the first term on page iBtPage. */ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ int bFlag; assert( pWriter->iBtPage || pWriter->nEmpty==0 ); if( pWriter->iBtPage==0 ) return; bFlag = fts5WriteFlushDlidx(p, pWriter); if( p->rc==SQLITE_OK ){ const char *z = (pWriter->btterm.n>0?(const char*)pWriter->btterm.p:""); /* The following was already done in fts5WriteInit(): */ /* sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); */ sqlite3_bind_blob(p->pIdxWriter, 2, z, pWriter->btterm.n, SQLITE_STATIC); sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); sqlite3_step(p->pIdxWriter); p->rc = sqlite3_reset(p->pIdxWriter); } pWriter->iBtPage = 0; } /* ** This is called once for each leaf page except the first that contains ** at least one term. Argument (nTerm/pTerm) is the split-key - a term that ** is larger than all terms written to earlier leaves, and equal to or ** smaller than the first term on the new leaf. ** ** If an error occurs, an error code is left in Fts5Index.rc. If an error ** has already occurred when this function is called, it is a no-op. */ static void fts5WriteBtreeTerm( Fts5Index *p, /* FTS5 backend object */ Fts5SegWriter *pWriter, /* Writer object */ int nTerm, const u8 *pTerm /* First term on new page */ ){ fts5WriteFlushBtree(p, pWriter); fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); pWriter->iBtPage = pWriter->writer.pgno; } /* ** This function is called when flushing a leaf page that contains no ** terms at all to disk. */ static void fts5WriteBtreeNoTerm( |
︙ | ︙ | |||
3340 3341 3342 3343 3344 3345 3346 | }else{ bDone = 1; } if( pDlidx->bPrevValid ){ iVal = iRowid - pDlidx->iPrev; }else{ | | | | 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 | }else{ bDone = 1; } if( pDlidx->bPrevValid ){ iVal = iRowid - pDlidx->iPrev; }else{ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, !bDone); sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iPgno); iVal = iRowid; } sqlite3Fts5BufferAppendVarint(&p->rc, &pDlidx->buf, iVal); pDlidx->bPrevValid = 1; pDlidx->iPrev = iRowid; } } static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; if( pWriter->bFirstTermInPage ){ /* No term was written to this page. */ assert( 0==fts5GetU16(&pPage->buf.p[2]) ); fts5WriteBtreeNoTerm(p, pWriter); } |
︙ | ︙ | |||
3394 3395 3396 3397 3398 3399 3400 | */ static void fts5WriteAppendTerm( Fts5Index *p, Fts5SegWriter *pWriter, int nTerm, const u8 *pTerm ){ int nPrefix; /* Bytes of prefix compression for term */ | | | 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 | */ static void fts5WriteAppendTerm( Fts5Index *p, Fts5SegWriter *pWriter, int nTerm, const u8 *pTerm ){ int nPrefix; /* Bytes of prefix compression for term */ Fts5PageWriter *pPage = &pWriter->writer; assert( pPage->buf.n==0 || pPage->buf.n>4 ); if( pPage->buf.n==0 ){ /* Zero the first term and first docid fields */ static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 }; fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero); assert( pWriter->bFirstTermInPage ); |
︙ | ︙ | |||
3429 3430 3431 3432 3433 3434 3435 | ** copy of (pTerm/nTerm) into the parent node. This is slightly ** inefficient, but still correct. */ int n = nTerm; if( pPage->term.n ){ n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); | | | 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 | ** copy of (pTerm/nTerm) into the parent node. This is slightly ** inefficient, but still correct. */ int n = nTerm; if( pPage->term.n ){ n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); pPage = &pWriter->writer; } }else{ nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, nTerm, pTerm); fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); } /* Append the number of bytes of new data, then the term data itself |
︙ | ︙ | |||
3467 3468 3469 3470 3471 3472 3473 | static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, i64 iRowid, int nPos ){ if( p->rc==SQLITE_OK ){ | | | 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 | static void fts5WriteAppendRowid( Fts5Index *p, Fts5SegWriter *pWriter, i64 iRowid, int nPos ){ if( p->rc==SQLITE_OK ){ Fts5PageWriter *pPage = &pWriter->writer; /* If this is to be the first docid written to the page, set the ** docid-pointer in the page-header. Also append a value to the dlidx ** buffer, in case a doclist-index is required. */ if( pWriter->bFirstRowidInPage ){ fts5PutU16(pPage->buf.p, pPage->buf.n); fts5WriteDlidxAppend(p, pWriter, iRowid); |
︙ | ︙ | |||
3502 3503 3504 3505 3506 3507 3508 | static void fts5WriteAppendPoslistData( Fts5Index *p, Fts5SegWriter *pWriter, const u8 *aData, int nData ){ | | | 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 | static void fts5WriteAppendPoslistData( Fts5Index *p, Fts5SegWriter *pWriter, const u8 *aData, int nData ){ Fts5PageWriter *pPage = &pWriter->writer; const u8 *a = aData; int n = nData; assert( p->pConfig->pgsz>0 ); while( p->rc==SQLITE_OK && (pPage->buf.n + n)>=p->pConfig->pgsz ){ int nReq = p->pConfig->pgsz - pPage->buf.n; int nCopy = 0; |
︙ | ︙ | |||
3525 3526 3527 3528 3529 3530 3531 | } if( n>0 ){ fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); } } static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ | | > < < < | < | < | | < < < < < < | < < < < | | < | < | < | < > | | > | > | < < | < < < < < | | < < < < < < < < < < | < < < < < < < < < | < | 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 | } if( n>0 ){ fts5BufferAppendBlob(&p->rc, &pPage->buf, n, a); } } static void fts5WriteAppendZerobyte(Fts5Index *p, Fts5SegWriter *pWriter){ fts5BufferAppendVarint(&p->rc, &pWriter->writer.buf, 0); } /* ** Flush any data cached by the writer object to the database. Free any ** allocations associated with the writer. */ static void fts5WriteFinish( Fts5Index *p, Fts5SegWriter *pWriter, /* Writer object */ int *pnHeight, /* OUT: Height of the b-tree */ int *pnLeaf /* OUT: Number of leaf pages in b-tree */ ){ int i; Fts5PageWriter *pLeaf = &pWriter->writer; if( p->rc==SQLITE_OK ){ if( pLeaf->pgno==1 && pLeaf->buf.n==0 ){ *pnLeaf = 0; *pnHeight = 0; }else{ if( pLeaf->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); } *pnLeaf = pLeaf->pgno-1; fts5WriteFlushBtree(p, pWriter); *pnHeight = 0; } } fts5BufferFree(&pLeaf->term); fts5BufferFree(&pLeaf->buf); fts5BufferFree(&pWriter->btterm); for(i=0; i<pWriter->nDlidx; i++){ sqlite3Fts5BufferFree(&pWriter->aDlidx[i].buf); } sqlite3_free(pWriter->aDlidx); } static void fts5WriteInit( Fts5Index *p, Fts5SegWriter *pWriter, int iSegid ){ memset(pWriter, 0, sizeof(Fts5SegWriter)); pWriter->iSegid = iSegid; fts5WriteDlidxGrow(p, pWriter, 1); pWriter->writer.pgno = 1; pWriter->bFirstTermInPage = 1; pWriter->iBtPage = 1; if( p->pIdxWriter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", pConfig->zDb, pConfig->zName )); } if( p->rc==SQLITE_OK ){ sqlite3_bind_int(p->pIdxWriter, 1, pWriter->iSegid); } } /* ** Iterator pIter was used to iterate through the input segments of on an ** incremental merge operation. This function is called if the incremental ** merge step has finished but the input has not been completely exhausted. |
︙ | ︙ | |||
3668 3669 3670 3671 3672 3673 3674 | fts5BufferZero(&buf); fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; | | | 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 | fts5BufferZero(&buf); fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); fts5BufferAppendBlob(&p->rc, &buf, pData->n - iOff, &pData->p[iOff]); fts5DataRelease(pData); pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 0, 1), iLeafRowid); fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } } fts5BufferFree(&buf); } |
︙ | ︙ | |||
3715 3716 3717 3718 3719 3720 3721 | memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ pLvlOut = &pStruct->aLevel[iLvl+1]; assert( pLvlOut->nSeg>0 ); nInput = pLvl->nMerge; | < > > > > | 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 | memset(&writer, 0, sizeof(Fts5SegWriter)); memset(&term, 0, sizeof(Fts5Buffer)); if( pLvl->nMerge ){ pLvlOut = &pStruct->aLevel[iLvl+1]; assert( pLvlOut->nSeg>0 ); nInput = pLvl->nMerge; pSeg = &pLvlOut->aSeg[pLvlOut->nSeg-1]; fts5WriteInit(p, &writer, pSeg->iSegid); writer.writer.pgno = pSeg->pgnoLast+1; writer.iBtPage = 0; }else{ int iSegid = fts5AllocateSegid(p, pStruct); /* Extend the Fts5Structure object as required to ensure the output ** segment exists. */ if( iLvl==pStruct->nLevel-1 ){ fts5StructureAddLevel(&p->rc, ppStruct); |
︙ | ︙ | |||
3807 3808 3809 3810 3811 3812 3813 | pLvl->nSeg -= nInput; pLvl->nMerge = 0; if( pSeg->pgnoLast==0 ){ pLvlOut->nSeg--; pStruct->nSegment--; } }else{ | | | 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 | pLvl->nSeg -= nInput; pLvl->nMerge = 0; if( pSeg->pgnoLast==0 ){ pLvlOut->nSeg--; pStruct->nSegment--; } }else{ assert( pSeg->pgnoLast>0 ); fts5TrimSegments(p, pIter); pLvl->nMerge = nInput; } fts5MultiIterFree(p, pIter); fts5BufferFree(&term); if( pnRem ) *pnRem -= writer.nLeafWritten; |
︙ | ︙ | |||
3982 3983 3984 3985 3986 3987 3988 | Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); /* Pre-allocate the buffer used to assemble leaf pages to the target ** page size. */ assert( pgsz>0 ); | | | 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 | Fts5SegWriter writer; fts5WriteInit(p, &writer, iSegid); /* Pre-allocate the buffer used to assemble leaf pages to the target ** page size. */ assert( pgsz>0 ); pBuf = &writer.writer.buf; fts5BufferGrow(&p->rc, pBuf, pgsz + 20); /* Begin scanning through hash table entries. This loop runs once for each ** term/doclist currently stored within the hash table. */ if( p->rc==SQLITE_OK ){ memset(pBuf->p, 0, 4); pBuf->n = 4; |
︙ | ︙ | |||
4006 4007 4008 4009 4010 4011 4012 | sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); nTerm = strlen(zTerm); /* Decide if the term will fit on the current leaf. If it will not, ** flush the leaf to disk here. */ if( (pBuf->n + nTerm + 2) > pgsz ){ fts5WriteFlushLeaf(p, &writer); | | | | | | 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 | sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); nTerm = strlen(zTerm); /* Decide if the term will fit on the current leaf. If it will not, ** flush the leaf to disk here. */ if( (pBuf->n + nTerm + 2) > pgsz ){ fts5WriteFlushLeaf(p, &writer); pBuf = &writer.writer.buf; if( (nTerm + 32) > pBuf->nSpace ){ fts5BufferGrow(&p->rc, pBuf, nTerm + 32 - pBuf->n); if( p->rc ) break; } } /* Write the term to the leaf. And if it is the first on the leaf, and ** the leaf is not page number 1, push it up into the b-tree hierarchy ** as well. */ if( writer.bFirstTermInPage==0 ){ int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nPre); nSuffix = nTerm - nPre; }else{ fts5PutU16(&pBuf->p[2], pBuf->n); writer.bFirstTermInPage = 0; if( writer.writer.pgno!=1 ){ int nPre = fts5PrefixCompress(nTerm, zPrev, nTerm, (const u8*)zTerm); fts5WriteBtreeTerm(p, &writer, nPre+1, (const u8*)zTerm); pBuf = &writer.writer.buf; assert( nPre<nTerm ); } nSuffix = nTerm; } pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], nSuffix); fts5BufferSafeAppendBlob(pBuf, (const u8*)&zTerm[nTerm-nSuffix], nSuffix); /* We just wrote a term into page writer.aWriter[0].pgno. If a ** doclist-index is to be generated for this doclist, it will be ** associated with this page. */ assert( writer.nDlidx>0 && writer.aDlidx[0].buf.n==0 ); writer.aDlidx[0].pgno = writer.writer.pgno; if( pgsz>=(pBuf->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ i64 iRowid = 0; i64 iDelta = 0; |
︙ | ︙ | |||
4095 4096 4097 4098 4099 4100 4101 | n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); } assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; if( pBuf->n>=pgsz ){ fts5WriteFlushLeaf(p, &writer); | | | 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 | n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); } assert( n>0 ); fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); iPos += n; if( pBuf->n>=pgsz ){ fts5WriteFlushLeaf(p, &writer); pBuf = &writer.writer.buf; } if( iPos>=nCopy ) break; } } iOff += nCopy; } } |
︙ | ︙ | |||
4128 4129 4130 4131 4132 4133 4134 | pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; pStruct->nSegment++; } fts5StructurePromote(p, 0, pStruct); } | < | 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 | pSeg->nHeight = nHeight; pSeg->pgnoFirst = 1; pSeg->pgnoLast = pgnoLast; pStruct->nSegment++; } fts5StructurePromote(p, 0, pStruct); } fts5IndexAutomerge(p, &pStruct, pgnoLast); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); } |
︙ | ︙ | |||
4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 | p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); if( p->zDataTbl && bCreate ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p); } } } assert( rc!=SQLITE_OK || p->rc==SQLITE_OK ); | > > > > > > | 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 | p->nWorkUnit = FTS5_WORK_UNIT; p->nMaxPendingData = 1024*1024; p->zDataTbl = sqlite3Fts5Mprintf(&rc, "%s_data", pConfig->zName); if( p->zDataTbl && bCreate ){ rc = sqlite3Fts5CreateTable( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable(pConfig, "idx", "segid, term, pgno, PRIMARY KEY(segid, term)", 1, pzErr ); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IndexReinit(p); } } } assert( rc!=SQLITE_OK || p->rc==SQLITE_OK ); |
︙ | ︙ | |||
4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 | */ int sqlite3Fts5IndexClose(Fts5Index *p){ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); sqlite3Fts5HashFree(p->pHash); sqlite3Fts5BufferFree(&p->scratch); sqlite3_free(p->zDataTbl); sqlite3_free(p); } return rc; } | > > > | 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 | */ int sqlite3Fts5IndexClose(Fts5Index *p){ int rc = SQLITE_OK; if( p ){ assert( p->pReader==0 ); sqlite3_finalize(p->pWriter); sqlite3_finalize(p->pDeleter); sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3Fts5HashFree(p->pHash); sqlite3Fts5BufferFree(&p->scratch); sqlite3_free(p->zDataTbl); sqlite3_free(p); } return rc; } |
︙ | ︙ | |||
4929 4930 4931 4932 4933 4934 4935 | ret += (ret<<3) + iCol; ret += (ret<<3) + iPos; if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i]; return ret; } | < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < < | 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 | ret += (ret<<3) + iCol; ret += (ret<<3) + iPos; if( iIdx>=0 ) ret += (ret<<3) + (FTS5_MAIN_PREFIX + iIdx); for(i=0; i<nTerm; i++) ret += (ret<<3) + pTerm[i]; return ret; } #ifdef SQLITE_DEBUG /* ** This function is purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. ** ** Instead, it tests that the same set of pgno/rowid combinations are ** visited regardless of whether the doclist-index identified by parameters |
︙ | ︙ | |||
5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 | p->rc = rc; } #else # define fts5TestDlidxReverse(x,y,z) # define fts5TestTerm(u,v,w,x,y,z) #endif static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | > > > > > > > > > < | < < > > > > > | | | | | | | | | | < < < | | | | 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 | p->rc = rc; } #else # define fts5TestDlidxReverse(x,y,z) # define fts5TestTerm(u,v,w,x,y,z) #endif /* ** Check that: ** ** 1) All leaves of pSeg between iFirst and iLast (inclusive) exist and ** contain zero terms. ** 2) All leaves of pSeg between iNoRowid and iLast (inclusive) exist and ** contain zero rowids. */ static void fts5IndexIntegrityCheckEmpty( Fts5Index *p, Fts5StructureSegment *pSeg, /* Segment to check internal consistency */ int iFirst, int iNoRowid, int iLast ){ int i; /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ for(i=iFirst; p->rc==SQLITE_OK && i<=iLast; i++){ Fts5Data *pLeaf = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, i)); if( pLeaf ){ if( 0!=fts5GetU16(&pLeaf->p[2]) ) p->rc = FTS5_CORRUPT; if( i>=iNoRowid && 0!=fts5GetU16(&pLeaf->p[0]) ) p->rc = FTS5_CORRUPT; } fts5DataRelease(pLeaf); if( p->rc ) break; } } static void fts5IndexIntegrityCheckSegment( Fts5Index *p, /* FTS5 backend object */ Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; int iDlidxPrevLeaf = pSeg->pgnoLast; if( pSeg->pgnoFirst==0 ) return; fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( "SELECT segid, term, (pgno>>1), (pgno & 1) FROM '%q'.'%q_idx' WHERE segid=%d", pConfig->zDb, pConfig->zName, pSeg->iSegid )); /* Iterate through the b-tree hierarchy. */ while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ int iOff; /* Offset of first term on leaf */ int i; /* Used to iterate through empty leaves */ int nIdxTerm = sqlite3_column_bytes(pStmt, 1); const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); int iIdxLeaf = sqlite3_column_int(pStmt, 2); int bIdxDlidx = sqlite3_column_int(pStmt, 3); /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ if( iIdxLeaf<pSeg->pgnoFirst ) continue; iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, 0, iIdxLeaf); pLeaf = fts5DataRead(p, iRow); if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal ** to or larger than the split-key in zIdxTerm. Also check that if there ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ iOff = fts5GetU16(&pLeaf->p[2]); if( iOff==0 ){ p->rc = FTS5_CORRUPT; }else{ int iRowidOff; int nTerm; /* Size of term on leaf in bytes */ int res; /* Comparison of term and split-key */ iRowidOff = fts5GetU16(&pLeaf->p[0]); if( iRowidOff>=iOff ){ p->rc = FTS5_CORRUPT; }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; if( res<0 ) p->rc = FTS5_CORRUPT; } } fts5DataRelease(pLeaf); if( p->rc ) break; /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ fts5IndexIntegrityCheckEmpty( p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 ); if( p->rc ) break; /* If there is a doclist-index, check that it looks right. */ if( bIdxDlidx ){ Fts5DlidxIter *pDlidx = 0; /* For iterating through doclist index */ int iPrevLeaf = iIdxLeaf; int iSegid = pSeg->iSegid; int iPg; i64 iKey; for(pDlidx=fts5DlidxIterInit(p, 0, iSegid, iIdxLeaf); fts5DlidxIterEof(p, pDlidx)==0; fts5DlidxIterNext(p, pDlidx) ){ /* Check any rowid-less pages that occur before the current leaf. */ for(iPg=iPrevLeaf+1; iPg<fts5DlidxIterPgno(pDlidx); iPg++){ iKey = FTS5_SEGMENT_ROWID(iSegid, 0, iPg); |
︙ | ︙ | |||
5264 5265 5266 5267 5268 5269 5270 | fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; } fts5DataRelease(pLeaf); } } | < < < | | | | > > | < | | | > > > | < | 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 | fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; } fts5DataRelease(pLeaf); } } iDlidxPrevLeaf = iPg; fts5DlidxIterFree(pDlidx); fts5TestDlidxReverse(p, iSegid, iIdxLeaf); }else{ iDlidxPrevLeaf = pSeg->pgnoLast; /* TODO: Check there is no doclist index */ } iIdxPrevLeaf = iIdxLeaf; } rc2 = sqlite3_finalize(pStmt); if( p->rc==SQLITE_OK ) p->rc = rc2; /* Page iter.iLeaf must now be the rightmost leaf-page in the segment */ #if 0 if( p->rc==SQLITE_OK && iter.iLeaf!=pSeg->pgnoLast ){ p->rc = FTS5_CORRUPT; } #endif } /* ** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums ** as calculated by fts5IndexEntryCksum() is cksum. |
︙ | ︙ |
Changes to ext/fts5/fts5_storage.c.
︙ | ︙ | |||
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | /* ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error ** code otherwise. */ int sqlite3Fts5DropAll(Fts5Config *pConfig){ int rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_data';" "DROP TABLE IF EXISTS %Q.'%q_config';", pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_docsize';", pConfig->zDb, pConfig->zName | > > | 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | /* ** Drop all shadow tables. Return SQLITE_OK if successful or an SQLite error ** code otherwise. */ int sqlite3Fts5DropAll(Fts5Config *pConfig){ int rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_data';" "DROP TABLE IF EXISTS %Q.'%q_idx';" "DROP TABLE IF EXISTS %Q.'%q_config';", pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_docsize';", pConfig->zDb, pConfig->zName |
︙ | ︙ | |||
214 215 216 217 218 219 220 221 222 223 224 225 226 227 | } int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ Fts5Config *pConfig = pStorage->pConfig; int rc = sqlite3Fts5StorageSync(pStorage, 1); fts5StorageRenameOne(pConfig, &rc, "data", zName); fts5StorageRenameOne(pConfig, &rc, "config", zName); if( pConfig->bColumnsize ){ fts5StorageRenameOne(pConfig, &rc, "docsize", zName); } if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ fts5StorageRenameOne(pConfig, &rc, "content", zName); } | > | 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | } int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ Fts5Config *pConfig = pStorage->pConfig; int rc = sqlite3Fts5StorageSync(pStorage, 1); fts5StorageRenameOne(pConfig, &rc, "data", zName); fts5StorageRenameOne(pConfig, &rc, "idx", zName); fts5StorageRenameOne(pConfig, &rc, "config", zName); if( pConfig->bColumnsize ){ fts5StorageRenameOne(pConfig, &rc, "docsize", zName); } if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ fts5StorageRenameOne(pConfig, &rc, "content", zName); } |
︙ | ︙ | |||
597 598 599 600 601 602 603 | */ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ Fts5Config *pConfig = p->pConfig; int rc; /* Delete the contents of the %_data and %_docsize tables. */ rc = fts5ExecPrintf(pConfig->db, 0, | | > > | 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 | */ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ Fts5Config *pConfig = p->pConfig; int rc; /* Delete the contents of the %_data and %_docsize tables. */ rc = fts5ExecPrintf(pConfig->db, 0, "DELETE FROM %Q.'%q_data';" "DELETE FROM %Q.'%q_idx';", pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName ); |
︙ | ︙ |
Changes to ext/fts5/test/fts5aa.test.
︙ | ︙ | |||
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 | do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; } { t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} } do_execsql_test 1.1 { DROP TABLE t1; SELECT name, sql FROM sqlite_master; } { } #------------------------------------------------------------------------- # reset_db do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); } do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } | > > | | 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); SELECT name, sql FROM sqlite_master; } { t1 {CREATE VIRTUAL TABLE t1 USING fts5(a, b, c)} t1_data {CREATE TABLE 't1_data'(id INTEGER PRIMARY KEY, block BLOB)} t1_idx {CREATE TABLE 't1_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID} t1_content {CREATE TABLE 't1_content'(id INTEGER PRIMARY KEY, c0, c1, c2)} t1_docsize {CREATE TABLE 't1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} t1_config {CREATE TABLE 't1_config'(k PRIMARY KEY, v) WITHOUT ROWID} } do_execsql_test 1.1 { DROP TABLE t1; SELECT name, sql FROM sqlite_master; } { } #------------------------------------------------------------------------- # reset_db do_execsql_test 2.0 { CREATE VIRTUAL TABLE t1 USING fts5(x,y); } do_execsql_test 2.1 { INSERT INTO t1 VALUES('a b c', 'd e f'); } do_test 2.2 { execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 } } {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=0 leaves=1..1}}}/} foreach w {a b c d e f} { do_execsql_test 2.3.$w.asc { SELECT rowid FROM t1 WHERE t1 MATCH $w; } {1} do_execsql_test 2.3.$w.desc { SELECT rowid FROM t1 WHERE t1 MATCH $w ORDER BY rowid DESC; |
︙ | ︙ |
Changes to ext/fts5/test/fts5content.test.
︙ | ︙ | |||
243 244 245 246 247 248 249 | #------------------------------------------------------------------------- # Check that a contentless table can be dropped. # reset_db do_execsql_test 6.1 { CREATE VIRTUAL TABLE xx USING fts5(x, y, content=""); SELECT name FROM sqlite_master; | | | 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 | #------------------------------------------------------------------------- # Check that a contentless table can be dropped. # reset_db do_execsql_test 6.1 { CREATE VIRTUAL TABLE xx USING fts5(x, y, content=""); SELECT name FROM sqlite_master; } {xx xx_data xx_idx xx_docsize xx_config} do_execsql_test 6.2 { DROP TABLE xx; SELECT name FROM sqlite_master; } {} finish_test |
︙ | ︙ |