Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | More optimization for the 'y' and 'b' matchinfo directives. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | fts3-matchinfo-y |
| Files: | files | file ages | folders |
| SHA1: |
8c5b9fedfcee3ac22a222819dceb981a |
| User & Date: | dan 2015-05-06 17:41:19 |
Context
|
2015-05-06
| ||
| 17:51 | Remove some dead code from fts3_snippet.c. (check-in: 46b2d3cef5 user: dan tags: fts3-matchinfo-y) | |
| 17:41 | More optimization for the 'y' and 'b' matchinfo directives. (check-in: 8c5b9fedfc user: dan tags: fts3-matchinfo-y) | |
| 08:43 | Further optimizations for the 'y' and 'b' matchinfo operators. (check-in: fbd038bb57 user: dan tags: fts3-matchinfo-y) | |
Changes
Changes to ext/fts3/fts3Int.h.
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 |
/* The following are used by the fts3_eval.c module. */
sqlite3_int64 iDocid; /* Current docid */
u8 bEof; /* True this expression is at EOF already */
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
u32 *aMI;
};
/*
** Candidate values for Fts3Query.eType. Note that the order of the first
** four values is in order of precedence when parsing expressions. For
** example, the following:
**
|
| > > |
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 |
/* The following are used by the fts3_eval.c module. */ sqlite3_int64 iDocid; /* Current docid */ u8 bEof; /* True this expression is at EOF already */ u8 bStart; /* True if iDocid is valid */ u8 bDeferred; /* True if this expression is entirely deferred */ /* The following are used by the fts3_snippet.c module. */ int iPhrase; /* Index of this phrase in matchinfo() results */ u32 *aMI; /* See above */ }; /* ** Candidate values for Fts3Query.eType. Note that the order of the first ** four values is in order of precedence when parsing expressions. For ** example, the following: ** |
Changes to ext/fts3/fts3_snippet.c.
283 284 285 286 287 288 289 290 291 292 293 294 295 296 ... 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 ... 850 851 852 853 854 855 856 857 858 859 860 861 862 863 .... 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 .... 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 .... 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 .... 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 .... 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 |
Fts3Expr *pExpr, /* Expression to iterate phrases of */
int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
void *pCtx /* Second argument to pass to callback */
){
int iPhrase = 0; /* Variable used as the phrase counter */
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** fts3ExprLoadDoclists().
*/
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
................................................................................
if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
if( pnToken ) *pnToken = sCtx.nToken;
return rc;
}
static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
(*(int *)ctx)++;
UNUSED_PARAMETER(pExpr);
UNUSED_PARAMETER(iPhrase);
return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
int nPhrase = 0;
(void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
return nPhrase;
}
................................................................................
c = *pEnd++ & 0x80;
if( !c ) nEntry++;
}
*ppCollist = pEnd;
return nEntry;
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
**
** fts3ExprIterate() callback to load the 'global' elements of a
** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
................................................................................
}
break;
case FTS3_MATCHINFO_LHITS_BM:
case FTS3_MATCHINFO_LHITS: {
int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
memset(pInfo->aMatchinfo, 0, nZero);
(void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
break;
}
default: {
Fts3Expr *pExpr;
assert( zArg[i]==FTS3_MATCHINFO_HITS );
pExpr = pCsr->pExpr;
................................................................................
}
/*
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(
sqlite3_context *pCtx, /* Return results here */
Fts3Cursor *pCsr, /* FTS3 Cursor object */
const char *zArg /* Second argument to matchinfo() function */
){
MatchInfo sInfo;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc = SQLITE_OK;
................................................................................
/* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
** matchinfo function has been called for this query. In this case
** allocate the array used to accumulate the matchinfo data and
** initialize those elements that are constant for every row.
*/
if( pCsr->pMIBuffer==0 ){
int nMatchinfo = 0; /* Number of u32 elements in match-info */
int nArg; /* Bytes in zArg */
int i; /* Used to iterate through zArg */
/* Determine the number of phrases in the query */
pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
sInfo.nPhrase = pCsr->nPhrase;
/* Determine the number of integers in the buffer returned by this call. */
................................................................................
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
if( xDestroyOut ) xDestroyOut(aOut);
}else{
int n = pCsr->pMIBuffer->nElem * sizeof(u32);
sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
}
return rc;
}
/*
** Implementation of snippet() function.
*/
void sqlite3Fts3Snippet(
sqlite3_context *pCtx, /* SQLite function call context */
................................................................................
void sqlite3Fts3Matchinfo(
sqlite3_context *pContext, /* Function call context */
Fts3Cursor *pCsr, /* FTS3 table cursor */
const char *zArg /* Second arg to matchinfo() function */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
const char *zFormat;
int rc;
if( zArg ){
zFormat = zArg;
}else{
zFormat = FTS3_MATCHINFO_DEFAULT;
}
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}else{
/* Retrieve matchinfo() data. */
rc = fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
}
}
#endif
|
> | < > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | | < < < < | |
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 ... 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 ... 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 .... 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 .... 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 .... 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 .... 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 .... 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 |
Fts3Expr *pExpr, /* Expression to iterate phrases of */
int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */
void *pCtx /* Second argument to pass to callback */
){
int iPhrase = 0; /* Variable used as the phrase counter */
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
** fts3ExprLoadDoclists().
*/
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
................................................................................
if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
if( pnToken ) *pnToken = sCtx.nToken;
return rc;
}
static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
(*(int *)ctx)++;
pExpr->iPhrase = iPhrase;
return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
int nPhrase = 0;
(void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
return nPhrase;
}
................................................................................
c = *pEnd++ & 0x80;
if( !c ) nEntry++;
}
*ppCollist = pEnd;
return nEntry;
}
/*
** This function gathers 'y' or 'b' data for a single phrase.
*/
static void fts3ExprLHits(
Fts3Expr *pExpr, /* Phrase expression node */
MatchInfo *p /* Matchinfo context */
){
Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
int iStart;
Fts3Phrase *pPhrase = pExpr->pPhrase;
char *pIter = pPhrase->doclist.pList;
int iCol = 0;
assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
if( p->flag==FTS3_MATCHINFO_LHITS ){
iStart = pExpr->iPhrase * p->nCol;
}else{
iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
}
while( 1 ){
int nHit = fts3ColumnlistCount(&pIter);
if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
if( p->flag==FTS3_MATCHINFO_LHITS ){
p->aMatchinfo[iStart + iCol] = (u32)nHit;
}else if( nHit ){
p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F));
}
}
assert( *pIter==0x00 || *pIter==0x01 );
if( *pIter!=0x01 ) break;
pIter++;
pIter += fts3GetVarint32(pIter, &iCol);
}
}
/*
** Gather the results for matchinfo directives 'y' and 'b'.
*/
static void fts3ExprLHitGather(
Fts3Expr *pExpr,
MatchInfo *p
){
assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
if( pExpr->pLeft ){
fts3ExprLHitGather(pExpr->pLeft, p);
fts3ExprLHitGather(pExpr->pRight, p);
}else{
fts3ExprLHits(pExpr, p);
}
}
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
**
** fts3ExprIterate() callback to load the 'global' elements of a
** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements
................................................................................
}
break;
case FTS3_MATCHINFO_LHITS_BM:
case FTS3_MATCHINFO_LHITS: {
int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
memset(pInfo->aMatchinfo, 0, nZero);
fts3ExprLHitGather(pCsr->pExpr, pInfo);
break;
}
default: {
Fts3Expr *pExpr;
assert( zArg[i]==FTS3_MATCHINFO_HITS );
pExpr = pCsr->pExpr;
................................................................................
}
/*
** Populate pCsr->aMatchinfo[] with data for the current row. The
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static void fts3GetMatchinfo(
sqlite3_context *pCtx, /* Return results here */
Fts3Cursor *pCsr, /* FTS3 Cursor object */
const char *zArg /* Second argument to matchinfo() function */
){
MatchInfo sInfo;
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
int rc = SQLITE_OK;
................................................................................
/* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
** matchinfo function has been called for this query. In this case
** allocate the array used to accumulate the matchinfo data and
** initialize those elements that are constant for every row.
*/
if( pCsr->pMIBuffer==0 ){
int nMatchinfo = 0; /* Number of u32 elements in match-info */
int i; /* Used to iterate through zArg */
/* Determine the number of phrases in the query */
pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
sInfo.nPhrase = pCsr->nPhrase;
/* Determine the number of integers in the buffer returned by this call. */
................................................................................
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
if( xDestroyOut ) xDestroyOut(aOut);
}else{
int n = pCsr->pMIBuffer->nElem * sizeof(u32);
sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
}
}
/*
** Implementation of snippet() function.
*/
void sqlite3Fts3Snippet(
sqlite3_context *pCtx, /* SQLite function call context */
................................................................................
void sqlite3Fts3Matchinfo(
sqlite3_context *pContext, /* Function call context */
Fts3Cursor *pCsr, /* FTS3 table cursor */
const char *zArg /* Second arg to matchinfo() function */
){
Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
const char *zFormat;
if( zArg ){
zFormat = zArg;
}else{
zFormat = FTS3_MATCHINFO_DEFAULT;
}
if( !pCsr->pExpr ){
sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
return;
}else{
/* Retrieve matchinfo() data. */
fts3GetMatchinfo(pContext, pCsr, zFormat);
sqlite3Fts3SegmentsClose(pTab);
}
}
#endif
|