Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
Comment: | Add the fts5vocab module, for direct access to the fts5 index. |
---|---|
Downloads: | Tarball | ZIP archive |
Timelines: | family | ancestors | descendants | both | fts5 |
Files: | files | file ages | folders |
SHA1: |
6bf93e3b56e6705b7d12bab5024fc615 |
User & Date: | dan 2015-05-08 20:21:24.206 |
Context
2015-05-09
| ||
18:28 | Allow the fts5vocab table to optionally provide data on a per-column basis. (check-in: 3922276135 user: dan tags: fts5) | |
2015-05-08
| ||
20:21 | Add the fts5vocab module, for direct access to the fts5 index. (check-in: 6bf93e3b56 user: dan tags: fts5) | |
09:21 | Improve the error message returned by FTS5 if it encounters an unknown file format. (check-in: f369caec14 user: dan tags: fts5) | |
Changes
Changes to ext/fts5/fts5.c.
︙ | ︙ | |||
141 142 143 144 145 146 147 | int aIdx[0]; /* Offsets into aPoslist for current row */ }; /* ** Virtual-table cursor object. ** | | | < < | | 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 | int aIdx[0]; /* Offsets into aPoslist for current row */ }; /* ** Virtual-table cursor object. ** ** iSpecial: ** If this is a 'special' query (refer to function fts5SpecialMatch()), ** then this variable contains the result of the query. */ struct Fts5Cursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ int idxNum; /* idxNum passed to xFilter() */ sqlite3_stmt *pStmt; /* Statement used to read %_content */ Fts5Expr *pExpr; /* Expression for MATCH queries */ Fts5Sorter *pSorter; /* Sorter for "ORDER BY rank" queries */ int csrflags; /* Mask of cursor flags (see below) */ Fts5Cursor *pNext; /* Next cursor in Fts5Cursor.pCsr list */ i64 iSpecial; /* Result of special query */ /* "rank" function. Populated on demand from vtab.xColumn(). */ char *zRank; /* Custom rank function */ char *zRankArgs; /* Custom rank function args */ Fts5Auxiliary *pRank; /* Rank callback (or NULL) */ int nRankArg; /* Number of trailing arguments for rank() */ sqlite3_value **apRankArg; /* Array of trailing arguments */ |
︙ | ︙ | |||
560 561 562 563 564 565 566 | /* Remove the cursor from the Fts5Global.pCsr list */ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; sqlite3_finalize(pCsr->pRankArgStmt); sqlite3_free(pCsr->apRankArg); | < | 558 559 560 561 562 563 564 565 566 567 568 569 570 571 | /* Remove the cursor from the Fts5Global.pCsr list */ for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext); *pp = pCsr->pNext; sqlite3_finalize(pCsr->pRankArgStmt); sqlite3_free(pCsr->apRankArg); if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){ sqlite3_free(pCsr->zRank); sqlite3_free(pCsr->zRankArgs); } sqlite3_free(pCsr); } return SQLITE_OK; |
︙ | ︙ | |||
795 796 797 798 799 800 801 | const char *z = zQuery; /* Special query text */ int n; /* Number of bytes in text at z */ while( z[0]==' ' ) z++; for(n=0; z[n] && z[n]!=' '; n++); assert( pTab->base.zErrMsg==0 ); | | | > > | < | 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 | const char *z = zQuery; /* Special query text */ int n; /* Number of bytes in text at z */ while( z[0]==' ' ) z++; for(n=0; z[n] && z[n]!=' '; n++); assert( pTab->base.zErrMsg==0 ); pCsr->idxNum = FTS5_PLAN_SPECIAL; if( 0==sqlite3_strnicmp("reads", z, n) ){ pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex); } else if( 0==sqlite3_strnicmp("id", z, n) ){ pCsr->iSpecial = pCsr->iCsrId; } else{ /* An unrecognized directive. Return an error message. */ pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); rc = SQLITE_ERROR; } |
︙ | ︙ | |||
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 | sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); }else{ fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); } } /* ** Return a "position-list blob" corresponding to the current position of ** cursor pCsr via sqlite3_result_blob(). A position-list blob contains ** the current position-list for each phrase in the query associated with ** cursor pCsr. ** ** A position-list blob begins with (nPhrase-1) varints, where nPhrase is | > > > > > > > > > > > > > > > > > > > > | 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 | sqlite3_result_error(context, zErr, -1); sqlite3_free(zErr); }else{ fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); } } /* ** Given cursor id iId, return a pointer to the corresponding Fts5Index ** object. Or NULL If the cursor id does not exist. */ Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ Fts5Cursor *pCsr; Fts5Index *pIndex = 0; for(pCsr=pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ if( pCsr->iCsrId==iCsrId ) break; } if( pCsr ){ Fts5Table *pTab = (Fts5Table*)pCsr->base.pVtab; pIndex = pTab->pIndex; } return pIndex; } /* ** Return a "position-list blob" corresponding to the current position of ** cursor pCsr via sqlite3_result_blob(). A position-list blob contains ** the current position-list for each phrase in the query associated with ** cursor pCsr. ** ** A position-list blob begins with (nPhrase-1) varints, where nPhrase is |
︙ | ︙ | |||
1724 1725 1726 1727 1728 1729 1730 | Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){ if( iCol==pConfig->nCol ){ | | | 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 | Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); if( pCsr->idxNum==FTS5_PLAN_SPECIAL ){ if( iCol==pConfig->nCol ){ sqlite3_result_int64(pCtx, pCsr->iSpecial); } }else if( iCol==pConfig->nCol ){ /* User is requesting the value of the special column with the same name ** as the table. Return the cursor integer id number. This value is only ** useful in that it may be passed as the first argument to an FTS5 |
︙ | ︙ | |||
2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 | pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 ); } } return rc; } #endif /* defined(SQLITE_ENABLE_FTS5) */ | > | 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 | pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); if( rc==SQLITE_OK ) rc = sqlite3Fts5AuxInit(&pGlobal->api); if( rc==SQLITE_OK ) rc = sqlite3Fts5TokenizerInit(&pGlobal->api); if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 ); } } return rc; } #endif /* defined(SQLITE_ENABLE_FTS5) */ |
Changes to ext/fts5/fts5Int.h.
︙ | ︙ | |||
47 48 49 50 51 52 53 | #ifdef SQLITE_TEST extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) #else # define assert_nc(x) assert(x) #endif | < < < < < < < < < < < < < < < < | 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | #ifdef SQLITE_TEST extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(sqlite3_fts5_may_be_corrupt || (x)) #else # define assert_nc(x) assert(x) #endif typedef struct Fts5Global Fts5Global; /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. */ typedef struct Fts5Config Fts5Config; |
︙ | ︙ | |||
256 257 258 259 260 261 262 263 264 265 266 267 268 269 | /* ** Values used as part of the flags argument passed to IndexQuery(). */ #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ /* ** Create/destroy an Fts5Index object. */ int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); int sqlite3Fts5IndexClose(Fts5Index *p); | > | 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 | /* ** Values used as part of the flags argument passed to IndexQuery(). */ #define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ #define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ #define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* ** Create/destroy an Fts5Index object. */ int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); int sqlite3Fts5IndexClose(Fts5Index *p); |
︙ | ︙ | |||
298 299 300 301 302 303 304 305 306 307 308 309 310 311 | i64 sqlite3Fts5IterRowid(Fts5IndexIter*); int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn); /* ** Close an iterator opened by sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter*); /* ** Insert or remove data to or from the index. Each time a document is ** added to or removed from the index, this function is called one or more ** times. ** ** For an insert, it must be called once for each token in the new document. | > > > > > > > | 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | i64 sqlite3Fts5IterRowid(Fts5IndexIter*); int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn); /* ** Close an iterator opened by sqlite3Fts5IndexQuery(). */ void sqlite3Fts5IterClose(Fts5IndexIter*); /* ** This interface is used by the fts5vocab module. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); int sqlite3Fts5IterNextScan(Fts5IndexIter*); /* ** Insert or remove data to or from the index. Each time a document is ** added to or removed from the index, this function is called one or more ** times. ** ** For an insert, it must be called once for each token in the new document. |
︙ | ︙ | |||
385 386 387 388 389 390 391 392 393 394 395 396 397 398 | #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) int sqlite3Fts5GetVarintLen(u32 iVal); /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; /* | > > > > > > > > > > > > > > > > > > > | 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 | #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) int sqlite3Fts5GetVarintLen(u32 iVal); /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5.c. */ int sqlite3Fts5GetTokenizer( Fts5Global*, const char **azArg, int nArg, Fts5Tokenizer**, fts5_tokenizer**, char **pzErr ); Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64); /* ** End of interface to code in fts5.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; /* |
︙ | ︙ | |||
602 603 604 605 606 607 608 609 610 | typedef struct Fts5Sorter Fts5Sorter; int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp); /* ** End of interface to code in fts5_sorter.c. **************************************************************************/ #endif | > > > > > > > > > > | 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 | typedef struct Fts5Sorter Fts5Sorter; int sqlite3Fts5SorterNew(Fts5Expr *pExpr, Fts5Sorter **pp); /* ** End of interface to code in fts5_sorter.c. **************************************************************************/ /************************************************************************** ** Interface to code in fts5_vocab.c. */ int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); /* ** End of interface to code in fts5_vocab.c. **************************************************************************/ #endif |
Changes to ext/fts5/fts5_index.c.
︙ | ︙ | |||
2071 2072 2073 2074 2075 2076 2077 | const u8 *pTerm, int nTerm, /* Term to seek to */ int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; int h; | | | 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 | const u8 *pTerm, int nTerm, /* Term to seek to */ int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5StructureSegment *pSeg, /* Description of segment */ Fts5SegIter *pIter /* Object to populate */ ){ int iPg = 1; int h; int bGe = (flags & FTS5INDEX_QUERY_SCAN); int bDlidx = 0; /* True if there is a doclist-index */ assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 ); assert( pTerm && nTerm ); memset(pIter, 0, sizeof(*pIter)); pIter->pSeg = pSeg; |
︙ | ︙ | |||
2167 2168 2169 2170 2171 2172 2173 | int nList = 0; const u8 *z = 0; int n = 0; assert( p->pHash ); assert( p->rc==SQLITE_OK ); | | | 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 | int nList = 0; const u8 *z = 0; int n = 0; assert( p->pHash ); assert( p->rc==SQLITE_OK ); if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); n = (z ? strlen((const char*)z) : 0); }else{ pIter->flags |= FTS5_SEGITER_ONETERM; sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); z = pTerm; |
︙ | ︙ | |||
4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 | Fts5Buffer *aBuf; const int nBuf = 32; aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ Fts5DoclistIter *pDoclist; int i; i64 iLastRowid = 0; Fts5MultiSegIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Buffer doclist; memset(&doclist, 0, sizeof(doclist)); | > | | 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 | Fts5Buffer *aBuf; const int nBuf = 32; aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); if( aBuf && pStruct ){ const int flags = FTS5INDEX_QUERY_SCAN; Fts5DoclistIter *pDoclist; int i; i64 iLastRowid = 0; Fts5MultiSegIter *p1 = 0; /* Iterator used to gather data from index */ Fts5Buffer doclist; memset(&doclist, 0, sizeof(doclist)); for(fts5MultiIterNew(p, pStruct, 1, flags, pToken, nToken, -1, 0, &p1); fts5MultiIterEof(p, p1)==0; fts5MultiIterNext(p, p1, 0, 0) ){ i64 iRowid = fts5MultiIterRowid(p1); int nTerm; const u8 *pTerm = fts5MultiIterTerm(p1, &nTerm); assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); |
︙ | ︙ | |||
4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 | int flags, /* Mask of FTS5INDEX_QUERY_X flags */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5IndexIter *pRet; int iIdx = 0; Fts5Buffer buf = {0, 0, 0}; if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); } if( flags & FTS5INDEX_QUERY_PREFIX ){ if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ | > > > > > | 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 | int flags, /* Mask of FTS5INDEX_QUERY_X flags */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5IndexIter *pRet; int iIdx = 0; Fts5Buffer buf = {0, 0, 0}; /* If the QUERY_SCAN flag is set, all other flags must be clear. */ assert( (flags & FTS5INDEX_QUERY_SCAN)==0 || (flags & FTS5INDEX_QUERY_SCAN)==FTS5INDEX_QUERY_SCAN ); if( sqlite3Fts5BufferGrow(&p->rc, &buf, nToken+1)==0 ){ memcpy(&buf.p[1], pToken, nToken); } if( flags & FTS5INDEX_QUERY_PREFIX ){ if( flags & FTS5INDEX_QUERY_TEST_NOIDX ){ |
︙ | ︙ | |||
4292 4293 4294 4295 4296 4297 4298 | memset(pRet, 0, sizeof(Fts5IndexIter)); pRet->pIndex = p; if( iIdx<=pConfig->nPrefix ){ buf.p[0] = FTS5_MAIN_PREFIX + iIdx; pRet->pStruct = fts5StructureRead(p); if( pRet->pStruct ){ | < | | 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 | memset(pRet, 0, sizeof(Fts5IndexIter)); pRet->pIndex = p; if( iIdx<=pConfig->nPrefix ){ buf.p[0] = FTS5_MAIN_PREFIX + iIdx; pRet->pStruct = fts5StructureRead(p); if( pRet->pStruct ){ fts5MultiIterNew( p, pRet->pStruct, 1, flags, buf.p, nToken+1, -1, 0, &pRet->pMulti ); } }else{ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pRet); } |
︙ | ︙ | |||
4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 | fts5DoclistIterNext(pIter->pDoclist); }else{ fts5BufferZero(&pIter->poslist); fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0); } return fts5IndexReturn(pIter->pIndex); } /* ** Move the doclist-iter passed as the first argument to the next ** matching rowid that occurs at or after iMatch. The definition of "at ** or after" depends on whether this iterator iterates in ascending or ** descending rowid order. */ | > > > > > > > > > > > > > > > > > > > > > > > | 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 | fts5DoclistIterNext(pIter->pDoclist); }else{ fts5BufferZero(&pIter->poslist); fts5MultiIterNext(pIter->pIndex, pIter->pMulti, 0, 0); } return fts5IndexReturn(pIter->pIndex); } /* ** Move to the next matching term/rowid. Used by the fts5vocab module. */ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIter){ Fts5Index *p = pIter->pIndex; Fts5MultiSegIter *pMulti = pIter->pMulti; assert( pIter->pIndex->rc==SQLITE_OK ); assert( pMulti ); fts5BufferZero(&pIter->poslist); fts5MultiIterNext(p, pMulti, 0, 0); if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; if( pSeg->pLeaf && pSeg->term.p[0]!=FTS5_MAIN_PREFIX ){ fts5DataRelease(pSeg->pLeaf); pSeg->pLeaf = 0; } } return fts5IndexReturn(pIter->pIndex); } /* ** Move the doclist-iter passed as the first argument to the next ** matching rowid that occurs at or after iMatch. The definition of "at ** or after" depends on whether this iterator iterates in ascending or ** descending rowid order. */ |
︙ | ︙ | |||
4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 | i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ if( pIter->pDoclist ){ return pIter->pDoclist->iRowid; }else{ return fts5MultiIterRowid(pIter->pMulti); } } /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer ** in bytes before returning. ** | > > > > > > > > > > | 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 | i64 sqlite3Fts5IterRowid(Fts5IndexIter *pIter){ if( pIter->pDoclist ){ return pIter->pDoclist->iRowid; }else{ return fts5MultiIterRowid(pIter->pMulti); } } /* ** Return the current term. */ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIter, int *pn){ int n; const char *z = fts5MultiIterTerm(pIter->pMulti, &n); *pn = n-1; return &z[1]; } /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer ** in bytes before returning. ** |
︙ | ︙ |
Added ext/fts5/fts5_vocab.c.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 | /* ** 2015 May 08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: ** ** May you do good and not evil. ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** ****************************************************************************** ** ** This is an SQLite module implementing full-text search. */ #if defined(SQLITE_ENABLE_FTS5) #include "fts5Int.h" typedef struct Fts5VocabTable Fts5VocabTable; typedef struct Fts5VocabCursor Fts5VocabCursor; struct Fts5VocabTable { sqlite3_vtab base; char *zFts5Tbl; /* Name of fts5 table */ char *zFts5Db; /* Db containing fts5 table */ sqlite3 *db; /* Database handle */ Fts5Global *pGlobal; /* FTS5 global object for this database */ }; struct Fts5VocabCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ Fts5Index *pIndex; /* Associated FTS5 index */ Fts5IndexIter *pIter; /* Iterator object */ int bEof; /* True if this cursor is at EOF */ Fts5Buffer term; /* Current value of 'term' column */ i64 nRow; /* Current value of 'row' column */ i64 nInst; /* Current value of 'inst' column */ i64 rowid; /* Current value of rowid column */ }; /* ** The xDisconnect() virtual table method. */ static int fts5VocabDisconnectMethod(sqlite3_vtab *pVtab){ Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; sqlite3_free(pTab); return SQLITE_OK; } /* ** The xDestroy() virtual table method. */ static int fts5VocabDestroyMethod(sqlite3_vtab *pVtab){ Fts5VocabTable *pTab = (Fts5VocabTable*)pVtab; sqlite3_free(pTab); return SQLITE_OK; } /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** ** The argv[] array contains the following: ** ** argv[0] -> module name ("fts5vocab") ** argv[1] -> database name ** argv[2] -> table name ** argv[3] -> name of fts5 table */ static int fts5VocabInitVtab( sqlite3 *db, /* The SQLite database connection */ void *pAux, /* Pointer to Fts5Global object */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ const char *zSchema = "CREATE TABLE vvv(term, row, inst)"; Fts5VocabTable *pRet = 0; int rc = SQLITE_OK; /* Return code */ if( argc!=4 ){ *pzErr = sqlite3_mprintf("wrong number of vtable arguments"); rc = SQLITE_ERROR; }else{ int nByte; /* Bytes of space to allocate */ const char *zDb = argv[1]; const char *zTab = argv[3]; int nDb = strlen(zDb) + 1; int nTab = strlen(zTab) + 1; rc = sqlite3_declare_vtab(db, zSchema); nByte = sizeof(Fts5VocabTable) + nDb + nTab; pRet = sqlite3Fts5MallocZero(&rc, nByte); if( pRet ){ pRet->pGlobal = (Fts5Global*)pAux; pRet->db = db; pRet->zFts5Tbl = (char*)&pRet[1]; pRet->zFts5Db = &pRet->zFts5Tbl[nTab]; memcpy(pRet->zFts5Tbl, zTab, nTab); memcpy(pRet->zFts5Db, zDb, nDb); } } *ppVTab = (sqlite3_vtab*)pRet; return rc; } /* ** The xConnect() and xCreate() methods for the virtual table. All the ** work is done in function fts5VocabInitVtab(). */ static int fts5VocabConnectMethod( sqlite3 *db, /* Database connection */ void *pAux, /* Pointer to tokenizer hash table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); } static int fts5VocabCreateMethod( sqlite3 *db, /* Database connection */ void *pAux, /* Pointer to tokenizer hash table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ char **pzErr /* OUT: sqlite3_malloc'd error message */ ){ return fts5VocabInitVtab(db, pAux, argc, argv, ppVtab, pzErr); } /* ** Implementation of the xBestIndex method. */ static int fts5VocabBestIndexMethod( sqlite3_vtab *pVTab, sqlite3_index_info *pInfo ){ return SQLITE_OK; } /* ** Implementation of xOpen method. */ static int fts5VocabOpenMethod( sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr ){ Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; Fts5VocabCursor *pCsr; int rc = SQLITE_OK; pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5VocabCursor)); if( pCsr ){ char *zSql = sqlite3_mprintf( "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl ); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); } sqlite3_free(zSql); if( rc==SQLITE_OK && sqlite3_step(pCsr->pStmt)==SQLITE_ROW ){ i64 iId = sqlite3_column_int64(pCsr->pStmt, 0); pCsr->pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId); } if( rc==SQLITE_OK && pCsr->pIndex==0 ){ rc = sqlite3_finalize(pCsr->pStmt); pCsr->pStmt = 0; if( rc==SQLITE_OK ){ pVTab->zErrMsg = sqlite3_mprintf( "no such fts5 table: %Q.%Q", pTab->zFts5Db, pTab->zFts5Tbl ); rc = SQLITE_ERROR; } } if( rc!=SQLITE_OK ){ sqlite3_free(pCsr); pCsr = 0; } } *ppCsr = (sqlite3_vtab_cursor*)pCsr; return rc; } static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); pCsr->pIter = 0; } /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. */ static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ if( pCursor ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; fts5VocabResetCursor(pCsr); sqlite3Fts5BufferFree(&pCsr->term); sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr); } return SQLITE_OK; } /* ** Advance the cursor to the next row in the table. */ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; int rc = SQLITE_OK; if( sqlite3Fts5IterEof(pCsr->pIter) ){ pCsr->bEof = 1; }else{ const char *zTerm; int nTerm; zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); pCsr->nInst = 0; pCsr->nRow = 0; pCsr->rowid++; while( 1 ){ const u8 *pPos; int nPos; /* Position list */ i64 dummy = 0; int iOff = 0; rc = sqlite3Fts5IterPoslist(pCsr->pIter, &pPos, &nPos); if( rc!=SQLITE_OK ) break; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &dummy) ){ pCsr->nInst++; } pCsr->nRow++; rc = sqlite3Fts5IterNextScan(pCsr->pIter); if( rc!=SQLITE_OK ) break; zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ) break; if( sqlite3Fts5IterEof(pCsr->pIter) ) break; } } return rc; } /* ** This is the xFilter implementation for the virtual table. */ static int fts5VocabFilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; int rc; const int flags = FTS5INDEX_QUERY_SCAN; fts5VocabResetCursor(pCsr); rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, &pCsr->pIter); if( rc==SQLITE_OK ){ rc = fts5VocabNextMethod(pCursor); } return rc; } /* ** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; return pCsr->bEof; } static int fts5VocabColumnMethod( sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; switch( iCol ){ case 0: /* term */ sqlite3_result_text( pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT ); break; case 1: /* row */ sqlite3_result_int64(pCtx, pCsr->nRow); break; case 2: /* inst */ sqlite3_result_int64(pCtx, pCsr->nInst); break; default: assert( 0 ); } return SQLITE_OK; } /* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts5 ** exposes %_content.docid as the rowid for the virtual table. The ** rowid should be written to *pRowid. */ static int fts5VocabRowidMethod( sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; *pRowid = pCsr->rowid; return SQLITE_OK; } int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ static const sqlite3_module fts5Vocab = { /* iVersion */ 2, /* xCreate */ fts5VocabCreateMethod, /* xConnect */ fts5VocabConnectMethod, /* xBestIndex */ fts5VocabBestIndexMethod, /* xDisconnect */ fts5VocabDisconnectMethod, /* xDestroy */ fts5VocabDestroyMethod, /* xOpen */ fts5VocabOpenMethod, /* xClose */ fts5VocabCloseMethod, /* xFilter */ fts5VocabFilterMethod, /* xNext */ fts5VocabNextMethod, /* xEof */ fts5VocabEofMethod, /* xColumn */ fts5VocabColumnMethod, /* xRowid */ fts5VocabRowidMethod, /* xUpdate */ 0, /* xBegin */ 0, /* xSync */ 0, /* xCommit */ 0, /* xRollback */ 0, /* xFindFunction */ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, }; void *p = (void*)pGlobal; return sqlite3_create_module_v2(db, "fts5vocab", &fts5Vocab, p, 0); } #endif /* defined(SQLITE_ENABLE_FTS5) */ |
Added ext/fts5/test/fts5vocab.test.
> > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 | # 2015 Apr 24 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: # # May you do good and not evil. # May you find forgiveness for yourself and forgive others. # May you share freely, never taking more than you give. # #*********************************************************************** # # The tests in this file focus on testing the fts5vocab module. # source [file join [file dirname [info script]] fts5_common.tcl] set testprefix fts5vocab do_execsql_test 1.1 { CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1); CREATE VIRTUAL TABLE v1 USING fts5vocab(t1); PRAGMA table_info = v1; } { 0 term {} 0 {} 0 1 row {} 0 {} 0 2 inst {} 0 {} 0 } do_execsql_test 1.2 { SELECT * FROM v1 } { } do_execsql_test 1.3 { INSERT INTO t1 VALUES('x y z'); INSERT INTO t1 VALUES('x x x'); } do_execsql_test 1.4 { SELECT * FROM v1; } {x 2 4 y 1 1 z 1 1} do_execsql_test 1.5 { BEGIN; INSERT INTO t1 VALUES('a b c'); SELECT * FROM v1 WHERE term<'d'; COMMIT; } {a 1 1 b 1 1 c 1 1} do_execsql_test 1.6 { DELETE FROM t1 WHERE one = 'a b c'; SELECT * FROM v1; } {x 2 4 y 1 1 z 1 1} finish_test |
Changes to main.mk.
︙ | ︙ | |||
78 79 80 81 82 83 84 85 86 87 88 89 90 91 | LIBOBJ += fts5_config.o LIBOBJ += fts5_expr.o LIBOBJ += fts5_hash.o LIBOBJ += fts5_index.o LIBOBJ += fts5_storage.o LIBOBJ += fts5_tokenize.o LIBOBJ += fts5_unicode2.o LIBOBJ += fts5parse.o # All of the source code files. # SRC = \ | > | 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | LIBOBJ += fts5_config.o LIBOBJ += fts5_expr.o LIBOBJ += fts5_hash.o LIBOBJ += fts5_index.o LIBOBJ += fts5_storage.o LIBOBJ += fts5_tokenize.o LIBOBJ += fts5_unicode2.o LIBOBJ += fts5_vocab.o LIBOBJ += fts5parse.o # All of the source code files. # SRC = \ |
︙ | ︙ | |||
242 243 244 245 246 247 248 | $(TOP)/ext/fts5/fts5_config.c \ $(TOP)/ext/fts5/fts5_expr.c \ $(TOP)/ext/fts5/fts5_hash.c \ $(TOP)/ext/fts5/fts5_index.c \ fts5parse.c fts5parse.h \ $(TOP)/ext/fts5/fts5_storage.c \ $(TOP)/ext/fts5/fts5_tokenize.c \ | | > | 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 | $(TOP)/ext/fts5/fts5_config.c \ $(TOP)/ext/fts5/fts5_expr.c \ $(TOP)/ext/fts5/fts5_hash.c \ $(TOP)/ext/fts5/fts5_index.c \ fts5parse.c fts5parse.h \ $(TOP)/ext/fts5/fts5_storage.c \ $(TOP)/ext/fts5/fts5_tokenize.c \ $(TOP)/ext/fts5/fts5_unicode2.c \ $(TOP)/ext/fts5/fts5_vocab.c # Generated source code files # SRC += \ keywordhash.h \ opcodes.c \ |
︙ | ︙ | |||
651 652 653 654 655 656 657 658 659 660 661 662 663 664 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c fts5_tokenize.o: $(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_tokenize.c fts5_unicode2.o: $(TOP)/ext/fts5/fts5_unicode2.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_unicode2.c fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h ./lemon $(OPTS) fts5parse.y mv fts5parse.c fts5parse.c.orig echo "#ifdef SQLITE_ENABLE_FTS5" > fts5parse.c | > > > | 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 | $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_storage.c fts5_tokenize.o: $(TOP)/ext/fts5/fts5_tokenize.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_tokenize.c fts5_unicode2.o: $(TOP)/ext/fts5/fts5_unicode2.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_unicode2.c fts5_vocab.o: $(TOP)/ext/fts5/fts5_vocab.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts5/fts5_vocab.c fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h ./lemon $(OPTS) fts5parse.y mv fts5parse.c fts5parse.c.orig echo "#ifdef SQLITE_ENABLE_FTS5" > fts5parse.c |
︙ | ︙ |