Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts3-matchinfo-y Excluding Merge-Ins
This is equivalent to a diff from 85a4a46c3b to 82e5a6e088
|
2015-05-11
| ||
| 19:01 | Add new fts3 matchinfo option 'b'. Also optimize existing option 'y'. (check-in: 2e7679a1df user: dan tags: trunk) | |
| 18:48 | Add missing "finish_test" commands to the end of the two new test scripts for sqlite3_analyzer and sqldiff. (check-in: 1d5e72b1c4 user: drh tags: trunk) | |
| 18:46 | Merge latest trunk changes into this branch. (Closed-Leaf check-in: 82e5a6e088 user: dan tags: fts3-matchinfo-y) | |
| 17:46 | Test cases for sqlite3_analyzer and sqldiff. Fix a problem with sqlite3_analyzer related to the renaming of the initialization routine. (check-in: 85a4a46c3b user: drh tags: trunk) | |
| 16:27 | Fix minor Makefile typos. Improve consistency of MSVC makefile. Add new targets to clean. (check-in: f84fbe9899 user: mistachkin tags: trunk) | |
|
2015-05-06
| ||
| 18:15 | Merge latest trunk changes with this branch. (check-in: 8a13e1fdbe user: dan tags: fts3-matchinfo-y) | |
Changes to ext/fts3/fts3.c.
1671 1671 static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ 1672 1672 Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; 1673 1673 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 1674 1674 sqlite3_finalize(pCsr->pStmt); 1675 1675 sqlite3Fts3ExprFree(pCsr->pExpr); 1676 1676 sqlite3Fts3FreeDeferredTokens(pCsr); 1677 1677 sqlite3_free(pCsr->aDoclist); 1678 - sqlite3_free(pCsr->aMatchinfo); 1678 + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 1679 1679 assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); 1680 1680 sqlite3_free(pCsr); 1681 1681 return SQLITE_OK; 1682 1682 } 1683 1683 1684 1684 /* 1685 1685 ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then ................................................................................ 3172 3172 if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; 3173 3173 if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; 3174 3174 assert( iIdx==nVal ); 3175 3175 3176 3176 /* In case the cursor has been used before, clear it now. */ 3177 3177 sqlite3_finalize(pCsr->pStmt); 3178 3178 sqlite3_free(pCsr->aDoclist); 3179 - sqlite3_free(pCsr->aMatchinfo); 3179 + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 3180 3180 sqlite3Fts3ExprFree(pCsr->pExpr); 3181 3181 memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); 3182 3182 3183 3183 /* Set the lower and upper bounds on docids to return */ 3184 3184 pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); 3185 3185 pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); 3186 3186
Changes to ext/fts3/fts3Int.h.
193 193 194 194 typedef struct Fts3Doclist Fts3Doclist; 195 195 typedef struct Fts3SegFilter Fts3SegFilter; 196 196 typedef struct Fts3DeferredToken Fts3DeferredToken; 197 197 typedef struct Fts3SegReader Fts3SegReader; 198 198 typedef struct Fts3MultiSegReader Fts3MultiSegReader; 199 199 200 +typedef struct MatchinfoBuffer MatchinfoBuffer; 201 + 200 202 /* 201 203 ** A connection to a fulltext index is an instance of the following 202 204 ** structure. The xCreate and xConnect methods create an instance 203 205 ** of this structure and xDestroy and xDisconnect free that instance. 204 206 ** All other methods receive a pointer to the structure as one of their 205 207 ** arguments. 206 208 */ ................................................................................ 302 304 u8 bDesc; /* True to sort in descending order */ 303 305 int eEvalmode; /* An FTS3_EVAL_XX constant */ 304 306 int nRowAvg; /* Average size of database rows, in pages */ 305 307 sqlite3_int64 nDoc; /* Documents in table */ 306 308 i64 iMinDocid; /* Minimum docid to return */ 307 309 i64 iMaxDocid; /* Maximum docid to return */ 308 310 int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ 309 - u32 *aMatchinfo; /* Information about most recent match */ 310 - int nMatchinfo; /* Number of elements in aMatchinfo[] */ 311 - char *zMatchinfo; /* Matchinfo specification */ 311 + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ 312 312 }; 313 313 314 314 #define FTS3_EVAL_FILTER 0 315 315 #define FTS3_EVAL_NEXT 1 316 316 #define FTS3_EVAL_MATCHINFO 2 317 317 318 318 /* ................................................................................ 424 424 425 425 /* The following are used by the fts3_eval.c module. */ 426 426 sqlite3_int64 iDocid; /* Current docid */ 427 427 u8 bEof; /* True this expression is at EOF already */ 428 428 u8 bStart; /* True if iDocid is valid */ 429 429 u8 bDeferred; /* True if this expression is entirely deferred */ 430 430 431 - u32 *aMI; 431 + /* The following are used by the fts3_snippet.c module. */ 432 + int iPhrase; /* Index of this phrase in matchinfo() results */ 433 + u32 *aMI; /* See above */ 432 434 }; 433 435 434 436 /* 435 437 ** Candidate values for Fts3Query.eType. Note that the order of the first 436 438 ** four values is in order of precedence when parsing expressions. For 437 439 ** example, the following: 438 440 ** ................................................................................ 560 562 561 563 /* fts3_snippet.c */ 562 564 void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); 563 565 void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, 564 566 const char *, const char *, int, int 565 567 ); 566 568 void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); 569 +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); 567 570 568 571 /* fts3_expr.c */ 569 572 int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, 570 573 char **, int, int, int, const char *, int, Fts3Expr **, char ** 571 574 ); 572 575 void sqlite3Fts3ExprFree(Fts3Expr *); 573 576 #ifdef SQLITE_TEST
Changes to ext/fts3/fts3_snippet.c.
24 24 #define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ 25 25 #define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ 26 26 #define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ 27 27 #define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ 28 28 #define FTS3_MATCHINFO_LCS 's' /* nCol values */ 29 29 #define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ 30 30 #define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ 31 +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ 31 32 32 33 /* 33 34 ** The default value for the second argument to matchinfo(). 34 35 */ 35 36 #define FTS3_MATCHINFO_DEFAULT "pcx" 36 37 37 38 ................................................................................ 85 86 */ 86 87 typedef struct MatchInfo MatchInfo; 87 88 struct MatchInfo { 88 89 Fts3Cursor *pCursor; /* FTS3 Cursor */ 89 90 int nCol; /* Number of columns in table */ 90 91 int nPhrase; /* Number of matchable phrases in query */ 91 92 sqlite3_int64 nDoc; /* Number of docs in database */ 93 + char flag; 92 94 u32 *aMatchinfo; /* Pre-allocated buffer */ 93 95 }; 94 96 97 +/* 98 +** An instance of this structure is used to manage a pair of buffers, each 99 +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below 100 +** for details. 101 +*/ 102 +struct MatchinfoBuffer { 103 + u8 aRef[3]; 104 + int nElem; 105 + int bGlobal; /* Set if global data is loaded */ 106 + char *zMatchinfo; 107 + u32 aMatchinfo[0]; 108 +}; 95 109 96 110 97 111 /* 98 112 ** The snippet() and offsets() functions both return text values. An instance 99 113 ** of the following structure is used to accumulate those values while the 100 114 ** functions are running. See fts3StringAppend() for details. 101 115 */ ................................................................................ 102 116 typedef struct StrBuffer StrBuffer; 103 117 struct StrBuffer { 104 118 char *z; /* Pointer to buffer containing string */ 105 119 int n; /* Length of z in bytes (excl. nul-term) */ 106 120 int nAlloc; /* Allocated size of buffer z in bytes */ 107 121 }; 108 122 123 + 124 +/************************************************************************* 125 +** Start of MatchinfoBuffer code. 126 +*/ 127 + 128 +/* 129 +** Allocate a two-slot MatchinfoBuffer object. 130 +*/ 131 +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ 132 + MatchinfoBuffer *pRet; 133 + int nByte = sizeof(u32) * (2*nElem + 2) + sizeof(MatchinfoBuffer); 134 + int nStr = strlen(zMatchinfo); 135 + 136 + pRet = sqlite3_malloc(nByte + nStr+1); 137 + if( pRet ){ 138 + memset(pRet, 0, nByte); 139 + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; 140 + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); 141 + pRet->nElem = nElem; 142 + pRet->zMatchinfo = ((char*)pRet) + nByte; 143 + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); 144 + pRet->aRef[0] = 1; 145 + } 146 + 147 + return pRet; 148 +} 149 + 150 +static void fts3MIBufferFree(void *p){ 151 + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); 152 + 153 + assert( (u32*)p==&pBuf->aMatchinfo[1] 154 + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] 155 + ); 156 + if( (u32*)p==&pBuf->aMatchinfo[1] ){ 157 + pBuf->aRef[1] = 0; 158 + }else{ 159 + pBuf->aRef[2] = 0; 160 + } 161 + 162 + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ 163 + sqlite3_free(pBuf); 164 + } 165 +} 166 + 167 +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ 168 + void (*xRet)(void*) = 0; 169 + u32 *aOut = 0; 170 + 171 + if( p->aRef[1]==0 ){ 172 + p->aRef[1] = 1; 173 + aOut = &p->aMatchinfo[1]; 174 + xRet = fts3MIBufferFree; 175 + } 176 + else if( p->aRef[2]==0 ){ 177 + p->aRef[2] = 1; 178 + aOut = &p->aMatchinfo[p->nElem+2]; 179 + xRet = fts3MIBufferFree; 180 + }else{ 181 + aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); 182 + if( aOut ){ 183 + xRet = sqlite3_free; 184 + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); 185 + } 186 + } 187 + 188 + *paOut = aOut; 189 + return xRet; 190 +} 191 + 192 +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ 193 + p->bGlobal = 1; 194 + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); 195 +} 196 + 197 +/* 198 +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() 199 +*/ 200 +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ 201 + if( p ){ 202 + assert( p->aRef[0]==1 ); 203 + p->aRef[0] = 0; 204 + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ 205 + sqlite3_free(p); 206 + } 207 + } 208 +} 209 + 210 +/* 211 +** End of MatchinfoBuffer code. 212 +*************************************************************************/ 213 + 109 214 110 215 /* 111 216 ** This function is used to help iterate through a position-list. A position 112 217 ** list is a list of unique integers, sorted from smallest to largest. Each 113 218 ** element of the list is represented by an FTS3 varint that takes the value 114 219 ** of the difference between the current element and the previous one plus 115 220 ** two. For example, to store the position-list: ................................................................................ 139 244 static int fts3ExprIterate2( 140 245 Fts3Expr *pExpr, /* Expression to iterate phrases of */ 141 246 int *piPhrase, /* Pointer to phrase counter */ 142 247 int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ 143 248 void *pCtx /* Second argument to pass to callback */ 144 249 ){ 145 250 int rc; /* Return code */ 146 - int eType = pExpr->eType; /* Type of expression node pExpr */ 251 + int eType = pExpr->eType; /* Type of expression node pExpr */ 147 252 148 253 if( eType!=FTSQUERY_PHRASE ){ 149 254 assert( pExpr->pLeft && pExpr->pRight ); 150 255 rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); 151 256 if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ 152 257 rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); 153 258 } ................................................................................ 172 277 Fts3Expr *pExpr, /* Expression to iterate phrases of */ 173 278 int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ 174 279 void *pCtx /* Second argument to pass to callback */ 175 280 ){ 176 281 int iPhrase = 0; /* Variable used as the phrase counter */ 177 282 return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); 178 283 } 284 + 179 285 180 286 /* 181 287 ** This is an fts3ExprIterate() callback used while loading the doclists 182 288 ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also 183 289 ** fts3ExprLoadDoclists(). 184 290 */ 185 291 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ ................................................................................ 217 323 if( pnPhrase ) *pnPhrase = sCtx.nPhrase; 218 324 if( pnToken ) *pnToken = sCtx.nToken; 219 325 return rc; 220 326 } 221 327 222 328 static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ 223 329 (*(int *)ctx)++; 224 - UNUSED_PARAMETER(pExpr); 225 - UNUSED_PARAMETER(iPhrase); 330 + pExpr->iPhrase = iPhrase; 226 331 return SQLITE_OK; 227 332 } 228 333 static int fts3ExprPhraseCount(Fts3Expr *pExpr){ 229 334 int nPhrase = 0; 230 335 (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); 231 336 return nPhrase; 232 337 } ................................................................................ 439 544 ** the set of phrases in the expression to populate the aPhrase[] array. 440 545 */ 441 546 sIter.pCsr = pCsr; 442 547 sIter.iCol = iCol; 443 548 sIter.nSnippet = nSnippet; 444 549 sIter.nPhrase = nList; 445 550 sIter.iCurrent = -1; 446 - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter); 551 + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); 447 552 if( rc==SQLITE_OK ){ 448 553 449 554 /* Set the *pmSeen output variable. */ 450 555 for(i=0; i<nList; i++){ 451 556 if( sIter.aPhrase[i].pHead ){ 452 557 *pmSeen |= (u64)1 << i; 453 558 } ................................................................................ 739 844 c = *pEnd++ & 0x80; 740 845 if( !c ) nEntry++; 741 846 } 742 847 743 848 *ppCollist = pEnd; 744 849 return nEntry; 745 850 } 851 + 852 +/* 853 +** This function gathers 'y' or 'b' data for a single phrase. 854 +*/ 855 +static void fts3ExprLHits( 856 + Fts3Expr *pExpr, /* Phrase expression node */ 857 + MatchInfo *p /* Matchinfo context */ 858 +){ 859 + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; 860 + int iStart; 861 + Fts3Phrase *pPhrase = pExpr->pPhrase; 862 + char *pIter = pPhrase->doclist.pList; 863 + int iCol = 0; 864 + 865 + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); 866 + if( p->flag==FTS3_MATCHINFO_LHITS ){ 867 + iStart = pExpr->iPhrase * p->nCol; 868 + }else{ 869 + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); 870 + } 871 + 872 + while( 1 ){ 873 + int nHit = fts3ColumnlistCount(&pIter); 874 + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ 875 + if( p->flag==FTS3_MATCHINFO_LHITS ){ 876 + p->aMatchinfo[iStart + iCol] = (u32)nHit; 877 + }else if( nHit ){ 878 + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); 879 + } 880 + } 881 + assert( *pIter==0x00 || *pIter==0x01 ); 882 + if( *pIter!=0x01 ) break; 883 + pIter++; 884 + pIter += fts3GetVarint32(pIter, &iCol); 885 + } 886 +} 887 + 888 +/* 889 +** Gather the results for matchinfo directives 'y' and 'b'. 890 +*/ 891 +static void fts3ExprLHitGather( 892 + Fts3Expr *pExpr, 893 + MatchInfo *p 894 +){ 895 + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); 896 + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ 897 + if( pExpr->pLeft ){ 898 + fts3ExprLHitGather(pExpr->pLeft, p); 899 + fts3ExprLHitGather(pExpr->pRight, p); 900 + }else{ 901 + fts3ExprLHits(pExpr, p); 902 + } 903 + } 904 +} 746 905 747 906 /* 748 907 ** fts3ExprIterate() callback used to collect the "global" matchinfo stats 749 908 ** for a single query. 750 909 ** 751 910 ** fts3ExprIterate() callback to load the 'global' elements of a 752 911 ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements ................................................................................ 806 965 p->aMatchinfo[iStart+i*3] = 0; 807 966 } 808 967 } 809 968 810 969 return rc; 811 970 } 812 971 813 -/* 814 -** fts3ExprIterate() callback used to gather information for the matchinfo 815 -** directive 'y'. 816 -*/ 817 -static int fts3ExprLHitsCb( 818 - Fts3Expr *pExpr, /* Phrase expression node */ 819 - int iPhrase, /* Phrase number */ 820 - void *pCtx /* Pointer to MatchInfo structure */ 821 -){ 822 - MatchInfo *p = (MatchInfo *)pCtx; 823 - Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; 824 - int rc = SQLITE_OK; 825 - int iStart = iPhrase * p->nCol; 826 - Fts3Expr *pEof; /* Ancestor node already at EOF */ 827 - 828 - /* This must be a phrase */ 829 - assert( pExpr->pPhrase ); 830 - 831 - /* Initialize all output integers to zero. */ 832 - memset(&p->aMatchinfo[iStart], 0, sizeof(u32) * p->nCol); 833 - 834 - /* Check if this or any parent node is at EOF. If so, then all output 835 - ** values are zero. */ 836 - for(pEof=pExpr; pEof && pEof->bEof==0; pEof=pEof->pParent); 837 - 838 - if( pEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ 839 - Fts3Phrase *pPhrase = pExpr->pPhrase; 840 - char *pIter = pPhrase->doclist.pList; 841 - int iCol = 0; 842 - 843 - while( 1 ){ 844 - int nHit = fts3ColumnlistCount(&pIter); 845 - if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ 846 - p->aMatchinfo[iStart + iCol] = (u32)nHit; 847 - } 848 - assert( *pIter==0x00 || *pIter==0x01 ); 849 - if( *pIter!=0x01 ) break; 850 - pIter++; 851 - pIter += fts3GetVarint32(pIter, &iCol); 852 - } 853 - } 854 - 855 - return rc; 856 -} 857 - 858 972 static int fts3MatchinfoCheck( 859 973 Fts3Table *pTab, 860 974 char cArg, 861 975 char **pzErr 862 976 ){ 863 977 if( (cArg==FTS3_MATCHINFO_NPHRASE) 864 978 || (cArg==FTS3_MATCHINFO_NCOL) 865 979 || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) 866 980 || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) 867 981 || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) 868 982 || (cArg==FTS3_MATCHINFO_LCS) 869 983 || (cArg==FTS3_MATCHINFO_HITS) 870 984 || (cArg==FTS3_MATCHINFO_LHITS) 985 + || (cArg==FTS3_MATCHINFO_LHITS_BM) 871 986 ){ 872 987 return SQLITE_OK; 873 988 } 874 989 sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); 875 990 return SQLITE_ERROR; 876 991 } 877 992 ................................................................................ 890 1005 case FTS3_MATCHINFO_LCS: 891 1006 nVal = pInfo->nCol; 892 1007 break; 893 1008 894 1009 case FTS3_MATCHINFO_LHITS: 895 1010 nVal = pInfo->nCol * pInfo->nPhrase; 896 1011 break; 1012 + 1013 + case FTS3_MATCHINFO_LHITS_BM: 1014 + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); 1015 + break; 897 1016 898 1017 default: 899 1018 assert( cArg==FTS3_MATCHINFO_HITS ); 900 1019 nVal = pInfo->nCol * pInfo->nPhrase * 3; 901 1020 break; 902 1021 } 903 1022 ................................................................................ 1085 1204 ){ 1086 1205 int rc = SQLITE_OK; 1087 1206 int i; 1088 1207 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; 1089 1208 sqlite3_stmt *pSelect = 0; 1090 1209 1091 1210 for(i=0; rc==SQLITE_OK && zArg[i]; i++){ 1092 - 1211 + pInfo->flag = zArg[i]; 1093 1212 switch( zArg[i] ){ 1094 1213 case FTS3_MATCHINFO_NPHRASE: 1095 1214 if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; 1096 1215 break; 1097 1216 1098 1217 case FTS3_MATCHINFO_NCOL: 1099 1218 if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; ................................................................................ 1145 1264 case FTS3_MATCHINFO_LCS: 1146 1265 rc = fts3ExprLoadDoclists(pCsr, 0, 0); 1147 1266 if( rc==SQLITE_OK ){ 1148 1267 rc = fts3MatchinfoLcs(pCsr, pInfo); 1149 1268 } 1150 1269 break; 1151 1270 1152 - case FTS3_MATCHINFO_LHITS: 1153 - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo); 1271 + case FTS3_MATCHINFO_LHITS_BM: 1272 + case FTS3_MATCHINFO_LHITS: { 1273 + int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); 1274 + memset(pInfo->aMatchinfo, 0, nZero); 1275 + fts3ExprLHitGather(pCsr->pExpr, pInfo); 1154 1276 break; 1277 + } 1155 1278 1156 1279 default: { 1157 1280 Fts3Expr *pExpr; 1158 1281 assert( zArg[i]==FTS3_MATCHINFO_HITS ); 1159 1282 pExpr = pCsr->pExpr; 1160 1283 rc = fts3ExprLoadDoclists(pCsr, 0, 0); 1161 1284 if( rc!=SQLITE_OK ) break; ................................................................................ 1180 1303 } 1181 1304 1182 1305 1183 1306 /* 1184 1307 ** Populate pCsr->aMatchinfo[] with data for the current row. The 1185 1308 ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). 1186 1309 */ 1187 -static int fts3GetMatchinfo( 1310 +static void fts3GetMatchinfo( 1311 + sqlite3_context *pCtx, /* Return results here */ 1188 1312 Fts3Cursor *pCsr, /* FTS3 Cursor object */ 1189 1313 const char *zArg /* Second argument to matchinfo() function */ 1190 1314 ){ 1191 1315 MatchInfo sInfo; 1192 1316 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; 1193 1317 int rc = SQLITE_OK; 1194 1318 int bGlobal = 0; /* Collect 'global' stats as well as local */ 1195 1319 1320 + u32 *aOut = 0; 1321 + void (*xDestroyOut)(void*) = 0; 1322 + 1196 1323 memset(&sInfo, 0, sizeof(MatchInfo)); 1197 1324 sInfo.pCursor = pCsr; 1198 1325 sInfo.nCol = pTab->nColumn; 1199 1326 1200 1327 /* If there is cached matchinfo() data, but the format string for the 1201 1328 ** cache does not match the format string for this request, discard 1202 1329 ** the cached data. */ 1203 - if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){ 1204 - assert( pCsr->aMatchinfo ); 1205 - sqlite3_free(pCsr->aMatchinfo); 1206 - pCsr->zMatchinfo = 0; 1207 - pCsr->aMatchinfo = 0; 1330 + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ 1331 + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); 1332 + pCsr->pMIBuffer = 0; 1208 1333 } 1209 1334 1210 - /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the 1335 + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the 1211 1336 ** matchinfo function has been called for this query. In this case 1212 1337 ** allocate the array used to accumulate the matchinfo data and 1213 1338 ** initialize those elements that are constant for every row. 1214 1339 */ 1215 - if( pCsr->aMatchinfo==0 ){ 1340 + if( pCsr->pMIBuffer==0 ){ 1216 1341 int nMatchinfo = 0; /* Number of u32 elements in match-info */ 1217 - int nArg; /* Bytes in zArg */ 1218 1342 int i; /* Used to iterate through zArg */ 1219 1343 1220 1344 /* Determine the number of phrases in the query */ 1221 1345 pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); 1222 1346 sInfo.nPhrase = pCsr->nPhrase; 1223 1347 1224 1348 /* Determine the number of integers in the buffer returned by this call. */ 1225 1349 for(i=0; zArg[i]; i++){ 1350 + char *zErr = 0; 1351 + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ 1352 + sqlite3_result_error(pCtx, zErr, -1); 1353 + sqlite3_free(zErr); 1354 + return; 1355 + } 1226 1356 nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); 1227 1357 } 1228 1358 1229 1359 /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ 1230 - nArg = (int)strlen(zArg); 1231 - pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1); 1232 - if( !pCsr->aMatchinfo ) return SQLITE_NOMEM; 1360 + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); 1361 + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; 1233 1362 1234 - pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo]; 1235 - pCsr->nMatchinfo = nMatchinfo; 1236 - memcpy(pCsr->zMatchinfo, zArg, nArg+1); 1237 - memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo); 1238 1363 pCsr->isMatchinfoNeeded = 1; 1239 1364 bGlobal = 1; 1240 1365 } 1241 1366 1242 - sInfo.aMatchinfo = pCsr->aMatchinfo; 1243 - sInfo.nPhrase = pCsr->nPhrase; 1244 - if( pCsr->isMatchinfoNeeded ){ 1367 + if( rc==SQLITE_OK ){ 1368 + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); 1369 + if( xDestroyOut==0 ){ 1370 + rc = SQLITE_NOMEM; 1371 + } 1372 + } 1373 + 1374 + if( rc==SQLITE_OK ){ 1375 + sInfo.aMatchinfo = aOut; 1376 + sInfo.nPhrase = pCsr->nPhrase; 1245 1377 rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); 1246 - pCsr->isMatchinfoNeeded = 0; 1378 + if( bGlobal ){ 1379 + fts3MIBufferSetGlobal(pCsr->pMIBuffer); 1380 + } 1247 1381 } 1248 1382 1249 - return rc; 1383 + if( rc!=SQLITE_OK ){ 1384 + sqlite3_result_error_code(pCtx, rc); 1385 + if( xDestroyOut ) xDestroyOut(aOut); 1386 + }else{ 1387 + int n = pCsr->pMIBuffer->nElem * sizeof(u32); 1388 + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); 1389 + } 1250 1390 } 1251 1391 1252 1392 /* 1253 1393 ** Implementation of snippet() function. 1254 1394 */ 1255 1395 void sqlite3Fts3Snippet( 1256 1396 sqlite3_context *pCtx, /* SQLite function call context */ ................................................................................ 1448 1588 1449 1589 /* Initialize the contents of sCtx.aTerm[] for column iCol. There is 1450 1590 ** no way that this operation can fail, so the return code from 1451 1591 ** fts3ExprIterate() can be discarded. 1452 1592 */ 1453 1593 sCtx.iCol = iCol; 1454 1594 sCtx.iTerm = 0; 1455 - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx); 1595 + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); 1456 1596 1457 1597 /* Retreive the text stored in column iCol. If an SQL NULL is stored 1458 1598 ** in column iCol, jump immediately to the next iteration of the loop. 1459 1599 ** If an OOM occurs while retrieving the data (this can happen if SQLite 1460 1600 ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 1461 1601 ** to the caller. 1462 1602 */ ................................................................................ 1540 1680 */ 1541 1681 void sqlite3Fts3Matchinfo( 1542 1682 sqlite3_context *pContext, /* Function call context */ 1543 1683 Fts3Cursor *pCsr, /* FTS3 table cursor */ 1544 1684 const char *zArg /* Second arg to matchinfo() function */ 1545 1685 ){ 1546 1686 Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; 1547 - int rc; 1548 - int i; 1549 1687 const char *zFormat; 1550 1688 1551 1689 if( zArg ){ 1552 - for(i=0; zArg[i]; i++){ 1553 - char *zErr = 0; 1554 - if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ 1555 - sqlite3_result_error(pContext, zErr, -1); 1556 - sqlite3_free(zErr); 1557 - return; 1558 - } 1559 - } 1560 1690 zFormat = zArg; 1561 1691 }else{ 1562 1692 zFormat = FTS3_MATCHINFO_DEFAULT; 1563 1693 } 1564 1694 1565 1695 if( !pCsr->pExpr ){ 1566 1696 sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); 1567 1697 return; 1568 - } 1569 - 1570 - /* Retrieve matchinfo() data. */ 1571 - rc = fts3GetMatchinfo(pCsr, zFormat); 1572 - sqlite3Fts3SegmentsClose(pTab); 1573 - 1574 - if( rc!=SQLITE_OK ){ 1575 - sqlite3_result_error_code(pContext, rc); 1576 1698 }else{ 1577 - int n = pCsr->nMatchinfo * sizeof(u32); 1578 - sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT); 1699 + /* Retrieve matchinfo() data. */ 1700 + fts3GetMatchinfo(pContext, pCsr, zFormat); 1701 + sqlite3Fts3SegmentsClose(pTab); 1579 1702 } 1580 1703 } 1581 1704 1582 1705 #endif
Changes to test/fts3matchinfo.test.
503 503 7 "a OR (a AND b)" { 504 504 1 {1 2 1 2 0 1} 2 {1 0 1 0 1 0} 3 {0 1 0 1 1 2} 4 {1 0 1 0 0 1} 505 505 5 {1 0 1 0 0 1} 6 {1 0 1 0 2 2} 7 {2 1 0 0 0 0} 8 {1 2 1 2 2 1} 506 506 9 {1 1 1 1 1 3} 10 {1 3 0 0 0 0} 507 507 } 508 508 509 509 } { 510 - do_execsql_test 11.1.$tn { 510 + do_execsql_test 11.1.$tn.1 { 511 511 SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr 512 512 } $res 513 + 514 + set r2 [list] 515 + foreach {rowid L} $res { 516 + lappend r2 $rowid 517 + set M [list] 518 + foreach {a b} $L { 519 + lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)] 520 + } 521 + lappend r2 $M 522 + } 523 + 524 + do_execsql_test 11.1.$tn.2 { 525 + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr 526 + } $r2 527 + breakpoint 528 + 529 + do_execsql_test 11.1.$tn.2 { 530 + SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr 531 + } $r2 513 532 } 514 533 set sqlite_fts3_enable_parentheses 0 515 534 535 +#--------------------------------------------------------------------------- 536 +# Test the 'b' matchinfo flag 537 +# 538 +set sqlite_fts3_enable_parentheses 1 539 +reset_db 540 +db func mit mit 541 + 542 +do_test 12.0 { 543 + set cols [list] 544 + for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" } 545 + execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])" 546 +} {} 547 + 548 +do_execsql_test 12.1 { 549 + INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc'); 550 + SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc'; 551 +} [list [list [expr 1<<4] [expr 1<<(45-32)]]] 552 + 553 +set sqlite_fts3_enable_parentheses 0 516 554 finish_test 555 +
Changes to test/fts3query.test.
169 169 } { 170 170 1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo 171 171 2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'" offsets 172 172 3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'" snippet 173 173 4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'" optimize 174 174 } 175 175 do_catchsql_test 5.5.1 { 176 - SELECT matchinfo(t2, 'abc') FROM t2 WHERE t2 MATCH 'history' 177 -} {1 {unrecognized matchinfo request: b}} 176 + SELECT matchinfo(t2, 'abcd') FROM t2 WHERE t2 MATCH 'history' 177 +} {1 {unrecognized matchinfo request: d}} 178 178 179 179 do_execsql_test 5.5 { DROP TABLE t2 } 180 180 181 181 182 182 # Test the snippet() function with 1 to 6 arguments. 183 183 # 184 184 do_execsql_test 6.1 {